diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 7e441c162a..49e94896cd 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 217a267 + GIT_TAG 9284147 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 7800fb1c88..54d2e046c2 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -116,7 +116,7 @@ Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the W 1. Download the macOS installation package. -2. Run the downloaded package to install TDengine. +2. Run the downloaded package to install TDengine. If the installation is blocked, you can right-click or ctrl-click on the installation package and select `Open`. @@ -179,12 +179,20 @@ The following `systemctl` commands can help you manage TDengine service: ::: +## Command Line Interface (CLI) + +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in terminal. + After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengine Server. +## Command Line Interface (CLI) + +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal to start the TDengine command line. + @@ -206,12 +214,12 @@ The following `launchctl` commands can help you manage TDengine service: ::: - - - ## Command Line Interface (CLI) -You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux/macOS terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line. +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in terminal. + + + ```bash taos diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 5ccddb06b0..45bbaa2751 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -15,10 +15,12 @@ import ConnCSNative from "./_connect_cs.mdx"; import ConnC from "./_connect_c.mdx"; import ConnR from "./_connect_r.mdx"; import ConnPHP from "./_connect_php.mdx"; -import InstallOnWindows from "../../14-reference/03-connector/_linux_install.mdx"; -import InstallOnLinux from "../../14-reference/03-connector/_windows_install.mdx"; +import InstallOnLinux from "../../14-reference/03-connector/_linux_install.mdx"; +import InstallOnWindows from "../../14-reference/03-connector/_windows_install.mdx"; +import InstallOnMacOS from "../../14-reference/03-connector/_macos_install.mdx"; import VerifyLinux from "../../14-reference/03-connector/_verify_linux.mdx"; import VerifyWindows from "../../14-reference/03-connector/_verify_windows.mdx"; +import VerifyMacOS from "../../14-reference/03-connector/_verify_macos.mdx"; Any application running on any platform can access TDengine through the REST API provided by TDengine. For information, see [REST API](/reference/rest-api/). Applications can also use the connectors for various programming languages, including C/C++, Java, Python, Go, Node.js, C#, and Rust, to access TDengine. These connectors support connecting to TDengine clusters using both native interfaces (taosc). Some connectors also support connecting over a REST interface. Community developers have also contributed several unofficial connectors, such as the ADO.NET connector, the Lua connector, and the PHP connector. @@ -44,10 +46,13 @@ If you are choosing to use the native connection and the the application is not - + - + + + + @@ -62,6 +67,9 @@ After the above installation and configuration are done and making sure TDengine + + + ## Install Connectors diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 2b39c4b9e5..6145d8c00c 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -863,6 +863,7 @@ INTERP(expr) - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. +- `INTERP` must be used along with `RANGE`, `EVERY`, `FILL` keywords. - The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. @@ -945,7 +946,7 @@ MIN(expr) MODE(expr) ``` -**Description**:The value which has the highest frequency of occurrence. NULL is returned if there are multiple values which have highest frequency of occurrence. +**Description**:The value which has the highest frequency of occurrence. One random value is returned if there are multiple values which have highest frequency of occurrence. **Return value type**: Same as the input data diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index f00e635af9..a33b302a92 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -7,7 +7,6 @@ title: TDengine Go Connector import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import Preparition from "./_preparation.mdx" import GoInsert from "../../07-develop/03-insert-data/_go_sql.mdx" import GoInfluxLine from "../../07-develop/03-insert-data/_go_line.mdx" import GoOpenTSDBTelnet from "../../07-develop/03-insert-data/_go_opts_telnet.mdx" @@ -176,6 +175,37 @@ func main() { } ``` + + +_taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver (driver-go minimum version 3.0.2). + +Use `taosWS` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN. + +* `writeTimeout` The timeout to send data via WebSocket. +* `readTimeout` The timeout to receive response data via WebSocket. + +For example: + +```go +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v3/taosWS" +) + +func main() { + var taosUri = "root:taosdata@ws(localhost:6041)/" + taos, err := sql.Open("taosWS", taosUri) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } +} +``` + ## Usage examples @@ -331,7 +361,7 @@ Creates consumer group. * `func (c *Consumer) Subscribe(topics []string) error` -Subscribes to a topic. +Subscribes to topics. * `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` @@ -409,6 +439,30 @@ Close consumer. Closes the parameter binding. +### Subscribe via WebSocket + +* `func NewConsumer(config *Config) (*Consumer, error)` + + Creates consumer group. + +* `func (c *Consumer) Subscribe(topic []string) error` + + Subscribes to topics. + +* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + + Polling information. + +* `func (c *Consumer) Commit(messageID uint64) error` + + Commit information. + +* `func (c *Consumer) Close() error` + + Close consumer. + +For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + ## API Reference Full API see [driver-go documentation](https://pkg.go.dev/github.com/taosdata/driver-go/v3) diff --git a/docs/en/14-reference/03-connector/_macos_install.mdx b/docs/en/14-reference/03-connector/_macos_install.mdx new file mode 100644 index 0000000000..effabbbebe --- /dev/null +++ b/docs/en/14-reference/03-connector/_macos_install.mdx @@ -0,0 +1,19 @@ +import PkgListV3 from "/components/PkgListV3"; + +1. Download the client installation package + + + + [All Downloads](../../releases/tdengine) + +2. Execute the installer, select the default value as prompted, and complete the installation. If the installation is blocked, you can right-click or ctrl-click on the installation package and select `Open`. +3. configure taos.cfg + + Edit `taos.cfg` file (full path is `/etc/taos/taos.cfg` by default), modify `firstEP` with actual TDengine server's End Point, for example `h1.tdengine.com:6030` + +:::tip + +1. If the computer does not run the TDengine service but installs the TDengine client driver, then you need to config `firstEP` in `taos.cfg` only, and there is no need to configure `FQDN`; +2. If you encounter the "Unable to resolve FQDN" error, please make sure the FQDN in the `/etc/hosts` file of the current computer is correctly configured, or the DNS service is correctly configured. + +::: diff --git a/docs/en/14-reference/03-connector/_verify_macos.mdx b/docs/en/14-reference/03-connector/_verify_macos.mdx new file mode 100644 index 0000000000..bac5f2f3e1 --- /dev/null +++ b/docs/en/14-reference/03-connector/_verify_macos.mdx @@ -0,0 +1,15 @@ +Execute TDengine CLI program `taos` directly from the macOS shell to connect to the TDengine service and enter the TDengine CLI interface, as shown in the following example. + +```text +$ taos + +taos> show databases; + name | +================================= + information_schema | + performance_schema | + db | +Query OK, 3 rows in database (0.019154s) + +taos> +``` diff --git a/docs/en/14-reference/03-connector/_windows_install.mdx b/docs/en/14-reference/03-connector/_windows_install.mdx index 81fc5573bb..a96e02feea 100644 --- a/docs/en/14-reference/03-connector/_windows_install.mdx +++ b/docs/en/14-reference/03-connector/_windows_install.mdx @@ -10,12 +10,14 @@ import PkgListV3 from "/components/PkgListV3"; The default installation path is C:\TDengine, including the following files (directories). - - _taos.exe_ : TDengine CLI command-line program - - _cfg_ : configuration file directory + - _taos.exe_: TDengine CLI command-line program + - _taosadapter.exe_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares + - _taosBenchmark.exe_: TDengine testing tool + - _cfg_: configuration file directory - _driver_: client driver dynamic link library - _examples_: sample programs bash/C/C#/go/JDBC/Python/Node.js - _include_: header files - - _log_ : log file + - _log_: log file - _unins000.exe_: uninstaller 4. configure taos.cfg diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 675a019cc1..4fd9c452d8 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -74,10 +74,12 @@ The different database framework specifications for various programming language import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -import InstallOnWindows from "./_linux_install.mdx"; -import InstallOnLinux from "./_windows_install.mdx"; +import InstallOnLinux from "./_linux_install.mdx"; +import InstallOnWindows from "./_windows_install.mdx"; +import InstallOnMacOS from "./_macos_install.mdx"; import VerifyWindows from "./_verify_windows.mdx"; import VerifyLinux from "./_verify_linux.mdx"; +import VerifyMacOS from "./_verify_macos.mdx"; ## Install Client Driver @@ -90,10 +92,13 @@ The client driver needs to be installed if you use the native interface connecto - + - + + + + @@ -108,5 +113,8 @@ After completing the above installation and configuration and you have confirmed + + + diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index a938084eb1..697ecb98a1 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -222,12 +222,12 @@ TDengine provides a variety of query processing functions for tables and STables ### Query Process -1. TDEngine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then check and verify the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. +1. TDengine client driver `taosc` parses the SQL statement and generates an abstract syntax tree (AST), then checks and verifies the AST according to metadata. During this stage, the metadata management module in `taosc` (Catalog) requests the metadata of the involved database and table from mnode and vnode. 2. After the verification passes, `taosc` generates distributed query plan and optimizes the plan. 3. `taosc` schedules the tasks according to configured query policy, a query sub-task may be scheduled to a vnode or qnode according to data relative and system load. Please be noted that both vnode and qnode are logic execution unit, the physical execution node is dnode (data node). 4. When a dnode receives a query request, it identifies which vnode or qnode this query request is targeted, and forwards the request to the query execution queue of the identified vnode or qnode. 5. The query execution thread of the vnode or qnode establishes fundamental query execution context, and executes the query, and notifies the client once obtaining a part of result data. -6. TDengine client driver `taosc` will initiates next level query tasks or obtain the result simply. +6. TDengine client driver `taosc` will initiate next level query tasks or obtain the result simply. ### Aggregation by Time Axis, Downsampling, Interpolation diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index cee036454e..836beb2417 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -1,6 +1,58 @@ import taos -from taos.tmq import TaosConsumer -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') -for msg in consumer: - for row in msg: - print(row) +from taos.tmq import * + +conn = taos.connect() + +print("init") +conn.execute("drop database if exists py_tmq") +conn.execute("create database if not exists py_tmq vgroups 2") +conn.select_db("py_tmq") +conn.execute( + "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)" +) +conn.execute("create table if not exists tb1 using stb1 tags(1)") +conn.execute("create table if not exists tb2 using stb1 tags(2)") +conn.execute("create table if not exists tb3 using stb1 tags(3)") + +print("create topic") +conn.execute("drop topic if exists topic_ctb_column") +conn.execute( + "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" +) + +print("build consumer") +conf = TaosTmqConf() +conf.set("group.id", "tg2") +conf.set("td.connect.user", "root") +conf.set("td.connect.pass", "taosdata") +conf.set("enable.auto.commit", "true") + + +def tmq_commit_cb_print(tmq, resp, offset, param=None): + print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") + + +conf.set_auto_commit_cb(tmq_commit_cb_print, None) +tmq = conf.new_consumer() + +print("build topic list") + +topic_list = TaosTmqList() +topic_list.append("topic_ctb_column") + +print("basic consume loop") +tmq.subscribe(topic_list) + +sub_list = tmq.subscription() + +print("subscribed topics: ", sub_list) + +while 1: + res = tmq.poll(1000) + if res: + topic = res.get_topic_name() + vg = res.get_vgroup_id() + db = res.get_db_name() + print(f"topic: {topic}\nvgroup id: {vg}\ndb: {db}") + for row in res: + print(row) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 167a68d3c8..781a65402c 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -115,7 +115,7 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。 1. 从列表中下载获得 pkg 安装程序； -2. 运行可执行程序来安装 TDengine。 +2. 运行可执行程序来安装 TDengine。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 @@ -178,12 +178,20 @@ Active: inactive (dead) ::: +## TDengine 命令行（CLI） + +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 + 安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 +## TDengine 命令行（CLI） + +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 + @@ -205,12 +213,12 @@ Active: inactive (dead) ::: - - - ## TDengine 命令行（CLI） -为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux、macOS 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 + + + ```bash taos diff --git a/docs/zh/07-develop/01-connect/index.md b/docs/zh/07-develop/01-connect/index.md index d3ffe7eba2..da31dec4c1 100644 --- a/docs/zh/07-develop/01-connect/index.md +++ b/docs/zh/07-develop/01-connect/index.md @@ -14,9 +14,11 @@ import ConnCSNative from "./_connect_cs.mdx"; import ConnC from "./_connect_c.mdx"; import ConnR from "./_connect_r.mdx"; import ConnPHP from "./_connect_php.mdx"; -import InstallOnWindows from "../../08-connector/_linux_install.mdx"; -import InstallOnLinux from "../../08-connector/_windows_install.mdx"; +import InstallOnLinux from "../../08-connector/_linux_install.mdx"; +import InstallOnWindows from "../../08-connector/_windows_install.mdx"; +import InstallOnMacOS from "../../08-connector/_macos_install.mdx"; import VerifyLinux from "../../08-connector/_verify_linux.mdx"; +import VerifyMacOS from "../../08-connector/_verify_macos.mdx"; import VerifyWindows from "../../08-connector/_verify_windows.mdx"; TDengine 提供了丰富的应用程序开发接口，为了便于用户快速开发自己的应用，TDengine 支持了多种编程语言的连接器，其中官方连接器包括支持 C/C++、Java、Python、Go、Node.js、C#、Rust、Lua（社区贡献）和 PHP （社区贡献）的连接器。这些连接器支持使用原生接口（taosc）和 REST 接口（部分语言暂不支持）连接 TDengine 集群。社区开发者也贡献了多个非官方连接器，例如 ADO.NET 连接器、Lua 连接器和 PHP 连接器。 @@ -43,10 +45,13 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 - + - + + + + @@ -61,6 +66,9 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 + + + ## 安装连接器 diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 515d1b030b..7a6058db3c 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -177,6 +177,37 @@ func main() { } ``` + + +_taosWS_ 通过 `WebSocket` 实现了 Go 的 `database/sql/driver` 接口。只需要引入驱动（driver-go 最低版本 3.0.2）就可以使用[`database/sql`](https://golang.org/pkg/database/sql/)的接口。 + +使用 `taosWS` 作为 `driverName` 并且使用一个正确的 [DSN](#DSN) 作为 `dataSourceName`，DSN 支持的参数： + +* `writeTimeout` 通过 WebSocket 发送数据的超时时间。 +* `readTimeout` 通过 WebSocket 接收响应数据的超时时间。 + +示例： + +```go +package main + +import ( + "database/sql" + "fmt" + + _ "github.com/taosdata/driver-go/v3/taosWS" +) + +func main() { + var taosUri = "root:taosdata@ws(localhost:6041)/" + taos, err := sql.Open("taosWS", taosUri) + if err != nil { + fmt.Println("failed to connect TDengine, err:", err) + return + } +} +``` + ## 使用示例 @@ -410,6 +441,30 @@ func main() { 结束参数绑定。 +### 通过 WebSocket 订阅 + +* `func NewConsumer(config *Config) (*Consumer, error)` + + 创建消费者。 + +* `func (c *Consumer) Subscribe(topic []string) error` + + 订阅主题。 + +* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + + 轮询消息。 + +* `func (c *Consumer) Commit(messageID uint64) error` + + 提交消息。 + +* `func (c *Consumer) Close() error` + + 关闭消费者。 + +完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) + ## API 参考 全部 API 见 [driver-go 文档](https://pkg.go.dev/github.com/taosdata/driver-go/v3) diff --git a/docs/zh/08-connector/_macos_install.mdx b/docs/zh/08-connector/_macos_install.mdx new file mode 100644 index 0000000000..081134dfee --- /dev/null +++ b/docs/zh/08-connector/_macos_install.mdx @@ -0,0 +1,19 @@ +import PkgListV3 from "/components/PkgListV3"; + +1. 下载客户端安装包 + + + + [所有下载](../../releases/tdengine) + +2. 执行安装程序，按提示选择默认值，完成安装。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 +3. 配置 taos.cfg + + 编辑 `taos.cfg` 文件（默认路径/etc/taos/taos.cfg），将 `firstEP` 修改为 TDengine 服务器的 End Point，例如：`h1.tdengine.com:6030` + +:::tip + +1. 如本机没有部署 TDengine 服务，仅安装了应用驱动，则 `taos.cfg` 中仅需配置 `firstEP`，无需在本机配置 `FQDN`。 +2. 为防止与服务器端连接时出现“Unable to resolve FQDN”错误，建议确认本机的 `/etc/hosts` 文件已经配置了服务器正确的 FQDN 值，或配置好了 DNS 服务。 + +::: diff --git a/docs/zh/08-connector/_verify_macos.mdx b/docs/zh/08-connector/_verify_macos.mdx new file mode 100644 index 0000000000..276a541210 --- /dev/null +++ b/docs/zh/08-connector/_verify_macos.mdx @@ -0,0 +1,15 @@ +在 macOS shell 下直接执行 `taos` 连接到 TDengine 服务，进入到 TDengine CLI 界面，示例如下： + +```text +$ taos + +taos> show databases; + name | +================================= + information_schema | + performance_schema | + db | +Query OK, 3 rows in database (0.019154s) + +taos> +``` diff --git a/docs/zh/08-connector/_windows_install.mdx b/docs/zh/08-connector/_windows_install.mdx index 3cd688e615..04874c8a86 100644 --- a/docs/zh/08-connector/_windows_install.mdx +++ b/docs/zh/08-connector/_windows_install.mdx @@ -12,6 +12,8 @@ import PkgListV3 from "/components/PkgListV3"; 默认安装路径为：C:\TDengine，其中包括以下文件(目录)： - _taos.exe_：TDengine CLI 命令行程序 + - _taosadapter.exe_：提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 + - _taosBenchmark.exe_：TDengine 测试程序 - _cfg_ : 配置文件目录 - _driver_: 应用驱动动态链接库 - _examples_: 示例程序 bash/C/C#/go/JDBC/Python/Node.js diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index f54470f742..e00e0b2fa2 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -74,10 +74,12 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; -import InstallOnWindows from "./_linux_install.mdx"; -import InstallOnLinux from "./_windows_install.mdx"; +import InstallOnLinux from "./_linux_install.mdx"; +import InstallOnWindows from "./_windows_install.mdx"; +import InstallOnMacOS from "./_macos_install.mdx"; import VerifyWindows from "./_verify_windows.mdx"; import VerifyLinux from "./_verify_linux.mdx"; +import VerifyMacOS from "./_verify_macos.mdx"; ## 安装客户端驱动 @@ -90,10 +92,13 @@ import VerifyLinux from "./_verify_linux.mdx"; - + - + + + + @@ -108,5 +113,8 @@ import VerifyLinux from "./_verify_linux.mdx"; + + + diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 9346c3f763..68c7fe6f30 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -864,8 +864,9 @@ INTERP(expr) - INTERP 用于在指定时间断面获取指定列的记录值，如果该时间断面不存在符合条件的行数据，那么会根据 FILL 参数的设定进行插值。 - INTERP 的输入数据为指定列的数据，可以通过条件语句（where 子句）来对原始列数据进行过滤，如果没有指定过滤条件则输入为全部数据。 +- INTERP 需要同时与 RANGE，EVERY 和 FILL 关键字一起使用。 - INTERP 的输出时间范围根据 RANGE(timestamp1,timestamp2)字段来指定，需满足 timestamp1<=timestamp2。其中 timestamp1（必选值）为输出时间范围的起始值，即如果 timestamp1 时刻符合插值条件则 timestamp1 为输出的第一条记录，timestamp2（必选值）为输出时间范围的结束值，即输出的最后一条记录的 timestamp 不能大于 timestamp2。 -- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。如果没有指定 EVERY，则默认窗口大小为无穷大，即从 timestamp1 开始只有一个窗口。 +- INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数，即从 timestamp1 开始每隔固定长度的时间（EVERY 值）进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 @@ -946,7 +947,7 @@ MIN(expr) MODE(expr) ``` -**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，输出NULL。 +**功能说明**：返回出现频率最高的值，若存在多个频率相同的最高值，则随机输出其中某个值。 **返回数据类型**：与输入数据类型一致。 diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 19adaad116..d3fc803c94 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -212,6 +212,7 @@ tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); int32_t code = tmq_list_append(topicList, "topicname"); if (code) { + tmq_list_destroy(topicList); return NULL; } return topicList; diff --git a/include/client/taos.h b/include/client/taos.h index 270b647a77..1182a9c2f7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -198,6 +198,7 @@ DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw(TAOS* taos, char* lines, int len, int32_t *totalRows, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ diff --git a/include/common/systable.h b/include/common/systable.h index 102465c8c2..8b29525db3 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -27,7 +27,7 @@ extern "C" { #define TSDB_INS_TABLE_MNODES "ins_mnodes" #define TSDB_INS_TABLE_MODULES "ins_modules" #define TSDB_INS_TABLE_QNODES "ins_qnodes" -#define TSDB_INS_TABLE_BNODES "ins_bnodes" +#define TSDB_INS_TABLE_BNODES "ins_bnodes" // no longer used #define TSDB_INS_TABLE_SNODES "ins_snodes" #define TSDB_INS_TABLE_CLUSTER "ins_cluster" #define TSDB_INS_TABLE_DATABASES "ins_databases" diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c32fe81335..32626ca233 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -44,12 +44,17 @@ enum { ) // clang-format on -typedef struct { +typedef struct SWinKey { uint64_t groupId; TSKEY ts; } SWinKey; -static inline int sWinKeyCmprImpl(const void* pKey1, const void* pKey2) { +typedef struct SSessionKey { + STimeWindow win; + uint64_t groupId; +} SSessionKey; + +static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) { SWinKey* pWin1 = (SWinKey*)pKey1; SWinKey* pWin2 = (SWinKey*)pKey2; @@ -69,7 +74,7 @@ static inline int sWinKeyCmprImpl(const void* pKey1, const void* pKey2) { } static inline int winKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { - return sWinKeyCmprImpl(pKey1, pKey2); + return winKeyCmprImpl(pKey1, pKey2); } typedef struct { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index fd0521af3b..2d196be44c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -51,7 +51,6 @@ extern int32_t tsMnodeShmSize; extern int32_t tsVnodeShmSize; extern int32_t tsQnodeShmSize; extern int32_t tsSnodeShmSize; -extern int32_t tsBnodeShmSize; extern int32_t tsNumOfShmThreads; // queue & threads diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9f21ee007f..e25fbe6201 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -95,7 +95,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QNODE, TSDB_MGMT_TABLE_SNODE, - TSDB_MGMT_TABLE_BNODE, + TSDB_MGMT_TABLE_BNODE, // no longer used TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_DB, TSDB_MGMT_TABLE_FUNC, @@ -1485,11 +1485,10 @@ int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, - SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq, SMCreateBnodeReq, SMDropBnodeReq, - SDCreateBnodeReq, SDDropBnodeReq; + SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq; -int32_t tSerializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); -int32_t tDeserializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); +int32_t tSerializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); +int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq); typedef struct { int8_t replica; @@ -1902,7 +1901,7 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) { if (pRebInfo == NULL) { return NULL; } - strcpy(pRebInfo->key, key); + tstrncpy(pRebInfo->key, key, TSDB_SUBSCRIBE_KEY_LEN); pRebInfo->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebInfo->lostConsumers == NULL) { goto _err; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 3f917ff0d1..1a8865bf83 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -74,8 +74,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_DROP_QNODE, "dnode-drop-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CREATE_SNODE, "dnode-create-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_SNODE, "dnode-drop-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_CREATE_BNODE, "dnode-create-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_DND_DROP_BNODE, "dnode-drop-bnode", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "drop-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "server-status", NULL, NULL) @@ -107,9 +107,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "create-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "alter-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "drop-snode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "create-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "alter-bnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "drop-bnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_BNODE, "create-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_ALTER_BNODE, "alter-bnode", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_DROP_BNODE, "drop-bnode", NULL, NULL) // no longer used TD_DEF_MSG_TYPE(TDMT_MND_CREATE_DB, "create-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_DB, "drop-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_USE_DB, "use-db", NULL, NULL) @@ -243,7 +243,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_MON_VM_INFO, "monitor-vinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_QM_INFO, "monitor-qinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_SM_INFO, "monitor-sinfo", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MON_BM_INFO, "monitor-binfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_VM_LOAD, "monitor-vload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_MM_LOAD, "monitor-mload", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MON_QM_LOAD, "monitor-qload", NULL, NULL) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h deleted file mode 100644 index d5958d432b..0000000000 --- a/include/dnode/bnode/bnode.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BNODE_H_ -#define _TD_BNODE_H_ - -#include "tmsgcb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SBnode SBnode; - -typedef struct { -#ifdef WINDOWS - size_t avoidCompilationErrors; -#endif -} SBnodeLoad; - -typedef struct { - SMsgCb msgCb; -} SBnodeOpt; - -/* ------------------------ SBnode ------------------------ */ -/** - * @brief Start one Bnode in Dnode. - * - * @param path Path of the bnode. - * @param pOption Option of the bnode. - * @return SBnode* The bnode object. - */ -SBnode *bndOpen(const char *path, const SBnodeOpt *pOption); - -/** - * @brief Stop Bnode in Dnode. - * - * @param pBnode The bnode object to close. - */ -void bndClose(SBnode *pBnode); - -/** - * @brief Get the statistical information of Bnode - * - * @param pBnode The bnode object. - * @param pLoad Statistics of the bnode. - * @return int32_t 0 for success, -1 for failure. - */ -int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad); - -/** - * @brief Process a query message. - * - * @param pBnode The bnode object. - * @param pMsgs The array of SRpcMsg - * @return int32_t 0 for success, -1 for failure - */ -int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BNODE_H_*/ \ No newline at end of file diff --git a/include/libs/monitor/monitor.h b/include/libs/monitor/monitor.h index 30254d61a9..12e465e0f8 100644 --- a/include/libs/monitor/monitor.h +++ b/include/libs/monitor/monitor.h @@ -80,7 +80,6 @@ typedef struct { int8_t has_mnode; int8_t has_qnode; int8_t has_snode; - int8_t has_bnode; SMonDiskDesc logdir; SMonDiskDesc tempdir; } SMonDnodeInfo; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index c57e9718b2..d0e1c3d258 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -525,7 +525,7 @@ typedef struct SDataInserterNode { SDataSinkNode sink; int32_t numOfTables; uint32_t size; - char* pData; + void* pData; } SDataInserterNode; typedef struct SQueryInserterNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 8d85c139df..5f2ce66f25 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -356,7 +356,7 @@ typedef struct SVgDataBlocks { SVgroupInfo vg; int32_t numOfTables; // number of tables in current submit block uint32_t size; - char* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... + void* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; typedef struct SVnodeModifOpStmt { diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 9445a0baa5..ec68e8ed71 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -25,6 +25,8 @@ extern "C" { typedef struct SStreamTask SStreamTask; +typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2); + // incremental state storage typedef struct { SStreamTask* pOwner; @@ -32,6 +34,7 @@ typedef struct { TTB* pStateDb; TTB* pFuncStateDb; TTB* pFillStateDb; // todo refactor + TTB* pSessionStateDb; TXN txn; int32_t number; } SStreamState; @@ -57,6 +60,19 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key); int32_t streamStateClear(SStreamState* pState); void streamStateSetNumber(SStreamState* pState, int32_t number); +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen); +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); +int32_t streamStateSessionClear(SStreamState* pState); +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); + +SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key); +SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key); + int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index de31a970df..adf244e32a 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -43,6 +43,7 @@ extern "C" { #define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_RECOV_SIZE_LIMIT (100 * WAL_SCAN_BUF_SIZE) typedef enum { TAOS_WAL_WRITE = 1, diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 293d9d17f8..d4e94d6173 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -49,9 +49,15 @@ extern SDiskSpace tsTempSpace; void osDefaultInit(); void osUpdate(); void osCleanup(); + bool osLogSpaceAvailable(); bool osDataSpaceAvailable(); bool osTempSpaceAvailable(); + +bool osLogSpaceSufficient(); +bool osDataSpaceSufficient(); +bool osTempSpaceSufficient(); + void osSetTimezone(const char *timezone); void osSetSystemLocale(const char *inLocale, const char *inCharSet); @@ -59,4 +65,4 @@ void osSetSystemLocale(const char *inLocale, const char *inCharSet); } #endif -#endif /*_TD_OS_ENV_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENV_H_*/ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 6bc0e0e7dd..bbddb539c6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -450,6 +450,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) +#define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/include/util/tdef.h b/include/util/tdef.h index e6d28ae341..bb7f908ad4 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -239,6 +239,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_COL_TAG_NUM (TSDB_MAX_COLUMNS + TSDB_MAX_TAGS) #define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_KEY_LEN 256 @@ -482,7 +483,6 @@ enum { #define QNODE_HANDLE -1 #define SNODE_HANDLE -2 #define VNODE_HANDLE -3 -#define BNODE_HANDLE -4 #define CLIENT_HANDLE -5 #define TSDB_CONFIG_OPTION_LEN 32 @@ -496,6 +496,9 @@ enum { #define MAX_NUM_STR_SIZE 40 +#define MAX_META_MSG_IN_BATCH 1048576 +#define MAX_META_BATCH_RSP_SIZE (1 * 1048576 * 1024) + #ifdef __cplusplus } #endif diff --git a/packaging/tools/com.taosdata.taosadapter.plist b/packaging/tools/com.taosdata.taosadapter.plist new file mode 100644 index 0000000000..2956b1ea5a --- /dev/null +++ b/packaging/tools/com.taosdata.taosadapter.plist @@ -0,0 +1,33 @@ + + + + + Label + com.tdengine.taosadapter + ProgramArguments + + /usr/local/bin/taosadapter + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taosadapter + + \ No newline at end of file diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 9694a89a8f..660bfd4fbf 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -69,7 +69,7 @@ prompt_force=0 initd_mod=0 service_mod=2 -if pidof systemd &>/dev/null; then +if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -575,11 +575,11 @@ function install_web() { function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : fi - if pidof tarbitrator &>/dev/null; then + if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then ${csudo}service tarbitratord stop || : fi @@ -759,7 +759,7 @@ function updateProduct() { echo -e "${GREEN}Start to update ${productName}...${NC}" # Stop the service if running - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then if ((${service_mod} == 0)); then ${csudo}systemctl stop ${serverName} || : elif ((${service_mod} == 1)); then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 254e7212fc..f7d677c982 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -265,7 +265,7 @@ function update_TDengine() { tar -zxf ${tarName} echo -e "${GREEN}Start to update ${productName} client...${NC}" # Stop the client shell if running - if pidof ${clientName} &> /dev/null; then + if ps aux | grep -v grep | grep ${clientName} &> /dev/null; then kill_client sleep 1 fi diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index b33cd478b8..3b6d610e88 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -2,4 +2,5 @@ TDengine is a high-efficient, scalable, high-available distributed time-series d To configure TDengine : edit /etc/taos/taos.cfg To start service : launchctl start com.tdengine.taosd +To start Taos Adapter : launchctl start com.tdengine.taosadapter To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index b56a5a203d..9034fd85f5 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -77,7 +77,7 @@ os_type=0 if [ "$osType" != "Darwin" ]; then initd_mod=0 - if pidof systemd &>/dev/null; then + if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -130,7 +130,6 @@ function kill_taosadapter() { } function kill_taosd() { - ps -ef | grep ${serverName} pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo}kill -9 $pid || : @@ -168,7 +167,7 @@ function install_bin() { ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -182,21 +181,21 @@ function install_bin() { ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : - [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor > /dev/null 2>&1 || : + [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : else ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : @@ -206,14 +205,14 @@ function install_bin() { ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx > /dev/null 2>&1 || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} > /dev/null 2>&1 || : fi } @@ -238,7 +237,7 @@ function install_jemalloc() { if [ -f "${binary_dir}/build/lib/libjemalloc.so.2" ]; then ${csudo}/usr/bin/install -c -d /usr/local/lib ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.so.2 /usr/local/lib - ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so + ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/lib [ -f ${binary_dir}/build/lib/libjemalloc.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/lib/libjemalloc.a /usr/local/lib @@ -274,8 +273,8 @@ function install_avro() { if [ -f "${binary_dir}/build/$1/libavro.so.23.0.0" ] && [ -d /usr/local/$1 ]; then ${csudo}/usr/bin/install -c -d /usr/local/$1 ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.so.23.0.0 /usr/local/$1 - ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 - ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so + ${csudo}ln -sf libavro.so.23.0.0 /usr/local/$1/libavro.so.23 > /dev/null 2>&1 + ${csudo}ln -sf libavro.so.23 /usr/local/$1/libavro.so > /dev/null 2>&1 ${csudo}/usr/bin/install -c -d /usr/local/$1 [ -f ${binary_dir}/build/$1/libavro.a ] && ${csudo}/usr/bin/install -c -m 755 ${binary_dir}/build/$1/libavro.a /usr/local/$1 @@ -304,11 +303,11 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber} - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so > /dev/null 2>&1 if [ -d "${lib64_link_dir}" ]; then - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 - ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 > /dev/null 2>&1 + ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so > /dev/null 2>&1 fi if [ -f ${binary_dir}/build/lib/libtaosws.so ]; then @@ -316,23 +315,23 @@ function install_lib() { ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so > /dev/null 2>&1 || : fi else ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \ - ${lib_link_dir}/libtaos.1.dylib || : + ${lib_link_dir}/libtaos.1.dylib > /dev/null 2>&1 || : - ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || : + ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib > /dev/null 2>&1 || : if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then ${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \ ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||: - ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || : + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib > /dev/null 2>&1 || : fi fi @@ -346,6 +345,7 @@ function install_lib() { } function install_header() { + ${csudo}mkdir -p ${inc_link_dir} ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ @@ -353,13 +353,13 @@ function install_header() { if [ -f ${binary_dir}/build/include/taosws.h ]; then ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||: - ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||: + ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h > /dev/null 2>&1 ||: fi - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h > /dev/null 2>&1 + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h > /dev/null 2>&1 ${csudo}chmod 644 ${install_main_dir}/include/* } @@ -374,7 +374,7 @@ function install_config() { ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} ${csudo}ln -s ${cfg_install_dir}/${configFile} \ - ${install_main_dir}/cfg/${configFile} + ${install_main_dir}/cfg/${configFile} > /dev/null 2>&1 else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} @@ -395,7 +395,7 @@ function install_taosadapter_config() { ${cfg_install_dir}/taosadapter.toml.${verNumber} || : [ -f ${cfg_install_dir}/taosadapter.toml ] && ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml \ - ${install_main_dir}/cfg/taosadapter.toml || : + ${install_main_dir}/cfg/taosadapter.toml > /dev/null 2>&1 || : else if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \ @@ -408,12 +408,12 @@ function install_taosadapter_config() { function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - ${csudo}ln -s ${log_dir} ${install_main_dir}/log + ${csudo}ln -s ${log_dir} ${install_main_dir}/log > /dev/null 2>&1 } function install_data() { ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir} - ${csudo}ln -s ${data_dir} ${install_main_dir}/data + ${csudo}ln -s ${data_dir} ${install_main_dir}/data > /dev/null 2>&1 } function install_connector() { @@ -437,7 +437,7 @@ function install_web() { } function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then ${csudo}service ${serverName} stop || : fi @@ -533,7 +533,11 @@ function install_taosadapter_service() { function install_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist - ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : } function install_service() { @@ -567,7 +571,7 @@ function update_TDengine() { echo -e "${GREEN}Start to update ${productName}...${NC}" # Stop the service if running - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then if ((${service_mod} == 0)); then ${csudo}systemctl stop ${serverName} || : elif ((${service_mod} == 1)); then @@ -614,11 +618,12 @@ function update_TDengine() { else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" @@ -665,11 +670,12 @@ function install_TDengine() { else if [ "$osType" != "Darwin" ]; then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" else echo -e "${GREEN_DARK}To start service ${NC}: launchctl start com.tdengine.taosd${NC}" + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: launchctl start com.tdengine.taosadapter${NC}" fi - [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index b729513b97..708a93c4fa 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -75,7 +75,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then +if ps aux | grep -v grep | grep systemd &> /dev/null; then service_mod=0 elif $(which service &> /dev/null); then service_mod=1 @@ -421,7 +421,7 @@ function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof taosd &> /dev/null; then + if ps aux | grep -v grep | grep taosd &> /dev/null; then ${csudo}service taosd stop || : fi @@ -512,6 +512,11 @@ function install_service_on_launchctl() { ${csudo}cp ${install_main_dir}/service/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist || : ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : fi + if [ -f ${install_main_dir}/service/com.taosdata.taosadapter.plist ]; then + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}cp ${install_main_dir}/service/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist || : + fi } function install_taosadapter_service() { diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 8dee9da988..0e96c71d5d 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -25,7 +25,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &> /dev/null; then +if ps aux | grep -v grep | grep systemd &> /dev/null; then service_mod=0 elif $(which service &> /dev/null); then service_mod=1 @@ -82,7 +82,7 @@ function clean_service_on_sysvinit() { #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" #${csudo}sed -i "\|${restart_config_str}|d" /etc/inittab || : - if pidof taosd &> /dev/null; then + if ps aux | grep -v grep | grep taosd &> /dev/null; then echo "TDengine taosd is running, stopping it..." ${csudo}service taosd stop || : fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index bace0bc869..8e5630ec2f 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -55,7 +55,7 @@ fi initd_mod=0 service_mod=2 -if pidof systemd &>/dev/null; then +if ps aux | grep -v grep | grep systemd &>/dev/null; then service_mod=0 elif $(which service &>/dev/null); then service_mod=1 @@ -171,12 +171,12 @@ function clean_service_on_systemd() { } function clean_service_on_sysvinit() { - if pidof ${serverName} &>/dev/null; then + if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then echo "${productName} ${serverName} is running, stopping it..." ${csudo}service ${serverName} stop || : fi - if pidof tarbitrator &>/dev/null; then + if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then echo "${productName} tarbitrator is running, stopping it..." ${csudo}service tarbitratord stop || : fi @@ -215,6 +215,8 @@ function clean_service_on_sysvinit() { function clean_service_on_launchctl() { ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : } function clean_service() { diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 54f7a949f3..7ab7fa8599 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -28,7 +28,7 @@ if command -v sudo > /dev/null; then fi function kill_client() { - if [ -n "$(pidof ${clientName})" ]; then + if [ -n "$(ps aux | grep -v grep | grep ${clientName})" ]; then ${csudo}kill -9 $pid || : fi } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index fc489f08af..b3c5565cfb 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -438,21 +438,18 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { - if (option != TSDB_OPTION_CONFIGDIR) { - taos_init(); // initialize global config - } else { + if (option == TSDB_OPTION_CONFIGDIR) { tstrncpy(configDir, str, PATH_MAX); tscInfo("set cfg:%s to %s", configDir, str); return 0; + } else { + taos_init(); // initialize global config } SConfig *pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { - case TSDB_OPTION_CONFIGDIR: - pItem = cfgGetItem(pCfg, "configDir"); - break; case TSDB_OPTION_SHELL_ACTIVITY_TIMER: pItem = cfgGetItem(pCfg, "shellActivityTimer"); break; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 096e2cdac8..fa0e3212a3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + tscTrace("conn %u hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { @@ -440,6 +440,7 @@ int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S } if (userNum <= 0) { + taosMemoryFree(users); return TSDB_CODE_SUCCESS; } @@ -476,6 +477,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } if (dbNum <= 0) { + taosMemoryFree(dbs); return TSDB_CODE_SUCCESS; } @@ -514,6 +516,7 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } if (stbNum <= 0) { + taosMemoryFree(stbs); return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 940dd745c8..45d2de4a7a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -186,7 +186,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, STscObj* pTscObj = (*pRequest)->pTscObj; if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, sizeof((*pRequest)->self))) { - tscError("%d failed to add to request container, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%" PRIx64 " failed to add to request container, reqId:0x%" PRIu64 ", conn:%" PRIx64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); @@ -199,7 +199,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (tsQueryUseNodeAllocator && !qIsInsertValuesSql((*pRequest)->sqlstr, (*pRequest)->sqlLen)) { if (TSDB_CODE_SUCCESS != nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) { - tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%d, %s", (*pRequest)->self, + tscError("%d failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); @@ -371,7 +371,7 @@ int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList) { pInfo->pQnodeList = taosArrayDup(pNodeList); taosArraySort(pInfo->pQnodeList, compareQueryNodeLoad); tscDebug("QnodeList updated in cluster 0x%" PRIx64 ", num:%d", pInfo->clusterId, - taosArrayGetSize(pInfo->pQnodeList)); + (int)taosArrayGetSize(pInfo->pQnodeList)); } taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -955,7 +955,12 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: if (!pRequest->validateOnly) { - code = execLocalCmd(pRequest, pQuery); + if (NULL == pQuery->pRoot) { + terrno = TSDB_CODE_INVALID_PARA; + code = terrno; + } else { + code = execLocalCmd(pRequest, pQuery); + } } break; case QUERY_EXEC_MODE_RPC: @@ -997,7 +1002,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue handleQueryExecRsp(pRequest); - if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } @@ -1130,7 +1135,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM break; } - if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { + // TODO weired responding code? + if (TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } } @@ -2253,7 +2259,10 @@ void syncQueryFn(void* param, void* res, int32_t code) { void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { if (sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; - fp(param, NULL, terrno); + if (fp) { + fp(param, NULL, terrno); + } + return; } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 7d8026f314..286fb85373 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -146,7 +146,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,13 +154,12 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL - : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -172,7 +171,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) : (const char *)tstrerror(pRequest->code); } } @@ -945,7 +944,6 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { if (pResultInfo->completed) { // it is a local executed query, no need to do async fetch if (QUERY_EXEC_MODE_LOCAL == pRequest->body.execMode) { - ASSERT(pResultInfo->numOfRows >= 0); if (pResultInfo->localResultFetched) { pResultInfo->numOfRows = 0; pResultInfo->current = 0; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index cdf977bea3..945562ef36 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -194,6 +194,10 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + if(strlen(usedbRsp.db) == 0){ + return TSDB_CODE_MND_DB_NOT_EXIST; + } + SName name = {0}; tNameFromString(&name, usedbRsp.db, T_NAME_ACCT | T_NAME_DB); @@ -288,8 +292,10 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) { tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); struct SCatalog* pCatalog = NULL; - catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (TSDB_CODE_SUCCESS == code) { + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); + } } taosMemoryFree(pMsg->pData); @@ -393,6 +399,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { + blockDataDestroy(pBlock); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a257335931..1dc319be94 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -410,6 +410,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -419,7 +420,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -524,6 +525,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -533,7 +535,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -556,6 +558,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; char* string = NULL; + cJSON* json = NULL; // decode void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -565,7 +568,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { goto _exit; } - cJSON* json = cJSON_CreateObject(); + json = cJSON_CreateObject(); if (json == NULL) { goto _exit; } @@ -684,7 +687,7 @@ end: static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; - SDecoder coder; + SDecoder coder = {0}; SMDropStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; @@ -1212,6 +1215,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; + SSubmitReq* subReq = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT); if (!pRequest) { @@ -1228,8 +1232,8 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; - strcpy(pName.dbname, pRequest->pDb); - strcpy(pName.tname, tbname); + tstrncpy(pName.dbname, pRequest->pDb, sizeof(pName.dbname)); + tstrncpy(pName.tname, tbname, sizeof(pName.tname)); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1278,7 +1282,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; int32_t totalLen = sizeof(SSubmitReq) + submitLen; - SSubmitReq* subReq = taosMemoryCalloc(1, totalLen); + subReq = taosMemoryCalloc(1, totalLen); SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); @@ -1352,6 +1356,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) if (NULL == pQuery) { uError("create SQuery error"); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(subReq); goto end; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; @@ -1390,6 +1395,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); + taosMemoryFree(subReq); return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 24da5f7b70..2c48eae59f 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -28,8 +28,8 @@ #define QUOTE '"' #define SLASH '\\' -#define JUMP_SPACE(sql) \ - while (*sql != '\0') { \ +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ if (*sql == SPACE) \ sql++; \ else \ @@ -299,6 +299,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool for (; i < taosArrayGetSize(cols); i++) { SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); return -1; } } @@ -430,7 +431,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { STableMeta *pTableMeta = NULL; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); SRequestConnInfo conn = {0}; conn.pTrans = info->taos->pAppInfo->pTransporter; @@ -874,7 +875,8 @@ static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArra kv->i = ts; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); + return TSDB_CODE_SUCCESS; } @@ -915,16 +917,17 @@ static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { return TSDB_CODE_TSC_INVALID_VALUE; } -static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSmlMsgBuf *msg) { +static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; elements->measure = sql; // parse measure - while (*sql != '\0') { + while (sql < sqlEnd) { if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, strlen(sql) + 1); + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; continue; } if (IS_COMMA(sql)) { @@ -948,7 +951,7 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm } else { if (*sql == COMMA) sql++; elements->tags = sql; - while (*sql != '\0') { + while (sql < sqlEnd) { if (IS_SPACE(sql)) { break; } @@ -959,10 +962,10 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm elements->measureTagsLen = sql - elements->measure; // parse cols - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) elements->cols = sql; bool isInQuote = false; - while (*sql != '\0') { + while (sql < sqlEnd) { if (IS_QUOTE(sql)) { isInQuote = !isInQuote; } @@ -982,10 +985,10 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm } // parse timestamp - JUMP_SPACE(sql) + JUMP_SPACE(sql, sqlEnd) elements->timestamp = sql; - while (*sql != '\0') { - if (*sql == SPACE) { + while (sql < sqlEnd) { + if (isspace(*sql)) { break; } sql++; @@ -995,8 +998,8 @@ static int32_t smlParseInfluxString(const char *sql, SSmlLineInfo *elements, SSm return TSDB_CODE_SUCCESS; } -static void smlParseTelnetElement(const char **sql, const char **data, int32_t *len) { - while (**sql != '\0') { +static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { + while (*sql < sqlEnd) { if (**sql != SPACE && !(*data)) { *data = *sql; } else if (**sql == SPACE && *data) { @@ -1007,19 +1010,20 @@ static void smlParseTelnetElement(const char **sql, const char **data, int32_t * } } -static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTableName, SHashObj *dumplicateKey, +static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; const char *sql = data; size_t childTableNameLen = strlen(tsSmlChildTableName); - while (*sql != '\0') { - JUMP_SPACE(sql) + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) if (*sql == '\0') break; const char *key = sql; int32_t keyLen = 0; // parse key - while (*sql != '\0') { + while (sql < sqlEnd) { if (*sql == SPACE) { smlBuildInvalidDataMsg(msg, "invalid data", sql); return TSDB_CODE_SML_INVALID_DATA; @@ -1044,7 +1048,7 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab // parse value const char *value = sql; int32_t valueLen = 0; - while (*sql != '\0') { + while (sql < sqlEnd) { // parse value if (*sql == SPACE) { break; @@ -1082,18 +1086,18 @@ static int32_t smlParseTelnetTags(const char *data, SArray *cols, char *childTab kv->length = valueLen; kv->type = TSDB_DATA_TYPE_NCHAR; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); } return TSDB_CODE_SUCCESS; } // format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTableInfo *tinfo, SArray *cols) { +static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, SArray *cols) { if (!sql) return TSDB_CODE_SML_INVALID_DATA; // parse metric - smlParseTelnetElement(&sql, &tinfo->sTableName, &tinfo->sTableNameLen); + smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; @@ -1102,7 +1106,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable // parse timestamp const char *timestamp = NULL; int32_t tLen = 0; - smlParseTelnetElement(&sql, ×tamp, &tLen); + smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); if (!timestamp || tLen == 0) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); return TSDB_CODE_SML_INVALID_DATA; @@ -1117,7 +1121,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable // parse value const char *value = NULL; int32_t valueLen = 0; - smlParseTelnetElement(&sql, &value, &valueLen); + smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); if (!value || valueLen == 0) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); return TSDB_CODE_TSC_INVALID_VALUE; @@ -1135,7 +1139,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, SSmlTable } // parse tags - ret = smlParseTelnetTags(sql, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); + ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); return ret; @@ -1368,10 +1372,22 @@ static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { SHashObj *s1 = *(SHashObj **)key1; SHashObj *s2 = *(SHashObj **)key2; - SSmlKv *kv1 = *(SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv *kv2 = *(SSmlKv **)taosHashGet(s2, TS, TS_LEN); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); + SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); + SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); + if(!kv1pp || !kv2pp){ + uError("smlKvTimeHashCompare kv is null"); + return -1; + } + SSmlKv *kv1 = *kv1pp; + SSmlKv *kv2 = *kv2pp; + if(!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv1"); + return -1; + } + if(!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP){ + uError("smlKvTimeHashCompare kv2"); + return -1; + } if (kv1->i < kv2->i) { return -1; } else if (kv1->i > kv2->i) { @@ -1735,7 +1751,7 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { kv->i = tsVal; kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->length = (int16_t)tDataTypes[kv->type].bytes; - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); return TSDB_CODE_SUCCESS; } @@ -1932,6 +1948,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { } static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { + if(!cols) return TSDB_CODE_OUT_OF_MEMORY; cJSON *metricVal = cJSON_GetObjectItem(root, "value"); if (metricVal == NULL) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1941,7 +1958,7 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { if (!kv) { return TSDB_CODE_OUT_OF_MEMORY; } - if (cols) taosArrayPush(cols, &kv); + taosArrayPush(cols, &kv); kv->key = VALUE; kv->keyLen = VALUE_LEN; @@ -1955,7 +1972,9 @@ static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, SSmlMsgBuf *msg) { int32_t ret = TSDB_CODE_SUCCESS; - + if (!pKVs){ + return TSDB_CODE_OUT_OF_MEMORY; + } cJSON *tags = cJSON_GetObjectItem(root, "tags"); if (tags == NULL || tags->type != cJSON_Object) { return TSDB_CODE_TSC_INVALID_JSON; @@ -1985,14 +2004,14 @@ static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableN return TSDB_CODE_TSC_INVALID_JSON; } memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); + tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); continue; } // add kv to SSmlKv SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (pKVs) taosArrayPush(pKVs, &kv); + taosArrayPush(pKVs, &kv); // key kv->keyLen = keyLen; @@ -2061,11 +2080,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo * } /************* TSDB_SML_JSON_PROTOCOL function end **************/ -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { +static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { SSmlLineInfo elements = {0}; uDebug("SML:0x%" PRIx64 " smlParseInfluxLine sql:%s, hello", info->id, sql); - int ret = smlParseInfluxString(sql, &elements, &info->msgBuf); + int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); return ret; @@ -2103,6 +2122,8 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { if (!oneTable) { tinfo = smlBuildTableInfo(); if (!tinfo) { + smlDestroyCols(cols); + if (info->dataFormat) taosArrayDestroy(cols); return TSDB_CODE_TSC_OUT_OF_MEMORY; } taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); @@ -2170,7 +2191,7 @@ static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql) { return TSDB_CODE_SUCCESS; } -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { +static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { int ret = TSDB_CODE_SUCCESS; SSmlTableInfo *tinfo = smlBuildTableInfo(); if (!tinfo) { @@ -2184,7 +2205,7 @@ static int32_t smlParseTelnetLine(SSmlHandle *info, void *data) { } if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, tinfo, cols); + ret = smlParseTelnetString(info, (const char *)data, (char*)data + len, tinfo, cols); } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); } else { @@ -2275,7 +2296,7 @@ static int32_t smlParseJSON(SSmlHandle *info, char *payload) { for (int32_t i = 0; i < payloadNum; ++i) { cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(root)) ? root : cJSON_GetArrayItem(root, i); - ret = smlParseTelnetLine(info, dataPoint); + ret = smlParseTelnetLine(info, dataPoint, -1); if (ret != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); goto end; @@ -2295,7 +2316,7 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlTableInfo *tableData = *oneTable; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - strcpy(pName.dbname, info->pRequest->pDb); + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); memcpy(pName.tname, tableData->childTableName, strlen(tableData->childTableName)); SRequestConnInfo conn = {0}; @@ -2364,10 +2385,14 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } -static int32_t smlParseLine(SSmlHandle *info, char *lines[], int numLines) { +static int32_t smlParseLine(SSmlHandle *info, char *lines[], char* rawLine, char* rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - code = smlParseJSON(info, *lines); + if(lines){ + code = smlParseJSON(info, *lines); + }else if(rawLine){ + code = smlParseJSON(info, rawLine); + } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseJSON failed:%s", info->id, *lines); return code; @@ -2376,28 +2401,46 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], int numLines) { } for (int32_t i = 0; i < numLines; ++i) { + char *tmp = NULL; + int len = 0; + if(lines){ + tmp = lines[i]; + len = strlen(tmp); + }else if(rawLine){ + tmp = rawLine; + while(rawLine < rawLineEnd){ + if(*(rawLine++) == '\n'){ + break; + } + len++; + } + if(info->protocol == TSDB_SML_LINE_PROTOCOL && tmp[0] == '#'){ // this line is comment + continue; + } + } + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, lines[i]); + code = smlParseInfluxLine(info, tmp, len); } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, lines[i]); + code = smlParseTelnetLine(info, tmp, len); } else { ASSERT(0); } if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, lines[i]); + uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } } return code; } -static int smlProcess(SSmlHandle *info, char *lines[], int numLines) { +static int smlProcess(SSmlHandle *info, char *lines[], char* rawLine, char* rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; int32_t retryNum = 0; info->cost.parseTime = taosGetTimestampUs(); - code = smlParseLine(info, lines, numLines); + code = smlParseLine(info, lines, rawLine, rawLineEnd, numLines); if (code != 0) { uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; @@ -2490,39 +2533,8 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { smlDestroyInfo(info); } -/** - * taos_schemaless_insert() parse and insert data points into database according to - * different protocol. - * - * @param $lines input array may contain multiple lines, each line indicates a data point. - * If protocol=2 is used input array should contain single JSON - * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert - * multiple data points in JSON format, should include them in $JSON_string - * as a JSON array. - * @param $numLines indicates how many data points in $lines. - * If protocol = 2 is used this param will be ignored as $lines should - * contain single JSON string. - * @param $protocol indicates which protocol to use for parsing: - * 0 - influxDB line protocol - * 1 - OpenTSDB telnet line protocol - * 2 - OpenTSDB JSON format protocol - * @return return zero for successful insertion. Otherwise return none-zero error code of - * failure reason. - * - */ - -TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } +TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision) { int batchs = 0; STscObj *pTscObj = request->pTscObj; @@ -2546,12 +2558,6 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr goto end; } - if (!lines) { - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2602,15 +2608,28 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr info->affectedRows = perBatch; info->pRequest->body.queryFp = smlInsertCallback; info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, perBatch); - lines += perBatch; + int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); + if(lines){ + lines += perBatch; + } + if(rawLine){ + int num = 0; + while(rawLine < rawLineEnd){ + if(*(rawLine++) == '\n'){ + num++; + } + if(num == perBatch){ + break; + } + } + } if (code != TSDB_CODE_SUCCESS) { info->pRequest->body.queryFp(info, req, code); } } tsem_wait(¶ms.sem); -end: + end: taosThreadSpinDestroy(¶ms.lock); tsem_destroy(¶ms.sem); // ((STscObj *)taos)->schemalessType = 0; @@ -2618,3 +2637,80 @@ end: uDebug("resultend:%s", request->msgBuf); return (TAOS_RES *)request; } + +/** + * taos_schemaless_insert() parse and insert data points into database according to + * different protocol. + * + * @param $lines input array may contain multiple lines, each line indicates a data point. + * If protocol=2 is used input array should contain single JSON + * string(e.g. char *lines[] = {"$JSON_string"}). If need to insert + * multiple data points in JSON format, should include them in $JSON_string + * as a JSON array. + * @param $numLines indicates how many data points in $lines. + * If protocol = 2 is used this param will be ignored as $lines should + * contain single JSON string. + * @param $protocol indicates which protocol to use for parsing: + * 0 - influxDB line protocol + * 1 - OpenTSDB telnet line protocol + * 2 - OpenTSDB JSON format protocol + * @return return zero for successful insertion. Otherwise return none-zero error code of + * failure reason. + * + */ + +TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision); +} + +TAOS_RES *taos_schemaless_insert_raw(TAOS* taos, char* lines, int len, int32_t *totalRows, int protocol, int precision){ + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT); + if (!request) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + + if (!lines || len <= 0) { + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "lines is null", NULL); + return (TAOS_RES *)request; + } + + int numLines = 0; + *totalRows = 0; + char *tmp = lines; + for(int i = 0; i < len; i++){ + if(lines[i] == '\n' || i == len - 1){ + numLines++; + if(tmp[0] != '#' || protocol != TSDB_SML_LINE_PROTOCOL){ //ignore comment + (*totalRows)++; + } + tmp = lines + i + 1; + } + } + return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision); +} diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 7a800bd334..81d0d616c9 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -152,7 +152,7 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.boundTags = tags; pStmt->bInfo.tagsCached = false; - strcpy(pStmt->bInfo.stbFName, sTableName); + tstrncpy(pStmt->bInfo.stbFName, sTableName, sizeof(pStmt->bInfo.stbFName)); return TSDB_CODE_SUCCESS; } @@ -201,6 +201,9 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if(!pSrc){ + return TSDB_CODE_OUT_OF_MEMORY; + } STableDataBlocks* pDst = NULL; STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 922cab23c8..0bc33739c6 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -210,6 +210,8 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; + /*char topicName[TSDB_TOPIC_FNAME_LEN];*/ + /*int32_t vgId;*/ } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -233,12 +235,12 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); + tstrncpy(conf->groupId, value, TSDB_CGROUP_LEN); return TMQ_CONF_OK; } if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); + tstrncpy(conf->clientId, value, 256); return TMQ_CONF_OK; } @@ -407,6 +409,14 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { return 0; } +static void tmqCommitRspCountDown(SMqCommitCbParamSet* pParamSet) { + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + if (waitingRspNum == 0) { + tmqCommitDone(pParamSet); + } +} + int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; @@ -420,18 +430,13 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { #endif taosMemoryFree(pParam->pOffset); - if (pBuf->pData) taosMemoryFree(pBuf->pData); + taosMemoryFree(pBuf->pData); /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, * pOffset->version);*/ - // count down waiting rsp - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); + tmqCommitRspCountDown(pParamSet); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } return 0; } @@ -452,11 +457,13 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, len, code); if (code < 0) { - ASSERT(0); return -1; } void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - if (buf == NULL) return -1; + if (buf == NULL) { + taosMemoryFree(pOffset); + return -1; + } ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); @@ -464,15 +471,24 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT SEncoder encoder; tEncoderInit(&encoder, abuf, len); tEncodeSTqOffset(&encoder, pOffset); + tEncoderClear(&encoder); // build param SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + taosMemoryFree(pOffset); + taosMemoryFree(buf); + return -1; + } pParam->params = pParamSet; pParam->pOffset = pOffset; // build send info SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { + taosMemoryFree(pOffset); + taosMemoryFree(buf); + taosMemoryFree(pParam); return -1; } pMsgSendInfo->msgInfo = (SDataBuf){ @@ -547,6 +563,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); goto FAIL; } goto HANDLE_RSP; @@ -565,6 +583,7 @@ HANDLE_RSP: tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); return code; } else { code = 0; @@ -577,17 +596,20 @@ FAIL: return 0; } -int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; - if (msg != NULL) { - return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); - } - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + if (async) { + if (automatic) { + tmq->commitCb(tmq, code, tmq->commitCbUserParam); + } else { + userCb(tmq, code, userParam); + } + } return -1; } @@ -625,43 +647,37 @@ int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t } } + // no request is sent if (pParamSet->totalRspNum == 0) { tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); return 0; } - int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); - ASSERT(waitingRspNum >= 0); - if (waitingRspNum == 0) { - tmqCommitDone(pParamSet); - } + // count down since waiting rsp num init as 1 + tmqCommitRspCountDown(pParamSet); if (!async) { tsem_wait(&pParamSet->rspSem); code = pParamSet->rspErr; tsem_destroy(&pParamSet->rspSem); taosMemoryFree(pParamSet); - } else { - code = 0; - } - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } - } - #if 0 - if (!async) { taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); - } #endif + } - return 0; + return code; +} + +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { + if (msg) { + return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); + } else { + return tmqCommitConsumerImpl(tmq, automatic, async, userCb, userParam); + } } void tmqAssignAskEpTask(void* param, void* tmrId) { @@ -709,6 +725,7 @@ void tmqSendHbReq(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { + taosMemoryFree(param); return; } int64_t consumerId = tmq->consumerId; @@ -721,6 +738,7 @@ void tmqSendHbReq(void* param, void* tmrId) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { taosMemoryFree(pReq); + goto OVER; } sendInfo->msgInfo = (SDataBuf){ .pData = pReq, @@ -870,8 +888,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("consumer %" PRId64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), - pTmq->groupId); + tscError("setting up new consumer failed since %s, consumer group %s", terrstr(), conf->groupId); return NULL; } @@ -939,10 +956,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { return NULL; } - int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); - *pRefId = pTmq->refId; - if (pTmq->hbBgEnable) { + int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); + *pRefId = pTmq->refId; pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pRefId, tmqMgmt.timer); } @@ -966,14 +982,14 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; + tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); + req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; - tscDebug("call tmq subscribe, consumer: %ld, topic num %d", tmq->consumerId, sz); - for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1061,9 +1077,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = 0; FAIL: if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); - if (code != 0 && buf) { - taosMemoryFree(buf); - } + taosMemoryFree(buf); + return code; } @@ -1101,7 +1116,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { - taosMemoryFree(pMsg->pData); tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index ab4442eca7..54778e87a7 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -44,7 +44,7 @@ TEST(testCase, smlParseInfluxString_Test) { char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, &elements, &msgBuf); + int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -63,14 +63,14 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_NE(ret, 0); // case 3 false tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); @@ -79,7 +79,7 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -98,7 +98,7 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -116,21 +116,21 @@ TEST(testCase, smlParseInfluxString_Test) { tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_EQ(ret, 0); // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, &elements, &msgBuf); + ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); ASSERT_NE(ret, 0); taosMemoryFree(sql); } @@ -542,7 +542,7 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i]); + int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); ASSERT_NE(ret, 0); } @@ -561,7 +561,7 @@ TEST(testCase, smlParseTelnetLine_diff_type_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -617,7 +617,7 @@ TEST(testCase, smlParseTelnetLine_json_error_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); ASSERT_NE(ret, 0); } @@ -653,7 +653,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -688,7 +688,7 @@ TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { }; int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i]); + ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); @@ -1002,7 +1002,7 @@ TEST(testCase, sml_col_4096_Test) { int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i]); + ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); if (ret != TSDB_CODE_SUCCESS) break; } ASSERT_NE(ret, 0); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index a45f7b2913..28a4732abd 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -62,12 +62,6 @@ static const SSysDbTableSchema snodesSchema[] = { {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, }; -static const SSysDbTableSchema bnodesSchema[] = { - {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, - {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, -}; - static const SSysDbTableSchema clusterSchema[] = { {.name = "id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -277,7 +271,6 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema), true}, {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema), true}, // {TSDB_INS_TABLE_SNODES, snodesSchema, tListLen(snodesSchema)}, - // {TSDB_INS_TABLE_BNODES, bnodesSchema, tListLen(bnodesSchema)}, {TSDB_INS_TABLE_CLUSTER, clusterSchema, tListLen(clusterSchema), true}, {TSDB_INS_TABLE_DATABASES, userDBSchema, tListLen(userDBSchema), false}, {TSDB_INS_TABLE_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema), false}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1028a899b6..62706db980 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1892,12 +1892,13 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) for (int32_t k = 0; k < colNum; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL) || !pColInfoData->pData) { len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); if (len >= size - 1) return dumpBuf; continue; } + + void* var = colDataGetData(pColInfoData, j); switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: memset(pBuf, 0, sizeof(pBuf)); @@ -1926,8 +1927,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_DOUBLE: - len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); - if (len >= size - 1) return dumpBuf; + // len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); + // if (len >= size - 1) return dumpBuf; break; case TSDB_DATA_TYPE_BOOL: len += snprintf(dumpBuf + len, size - len, " %15d |", *(bool*)var); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad5772e0fe..7967b6885a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -602,7 +602,7 @@ _exit: } int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { - int32_t n; + int32_t n = 0; TSROW_LEN(pRow, n); if (p) { @@ -613,7 +613,7 @@ int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow) { } int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { - int32_t n; + int32_t n = 0; *ppRow = (STSRow2 *)p; TSROW_LEN(*ppRow, n); @@ -916,6 +916,10 @@ char *tTagValToData(const STagVal *value, bool isJson) { } bool tTagGet(const STag *pTag, STagVal *pTagVal) { + if(!pTag || !pTagVal){ + return false; + } + int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 56daa4075a..2b64d3f1ee 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -46,7 +46,6 @@ int32_t tsMnodeShmSize = TSDB_MAX_MSG_SIZE * 2 + 1024; int32_t tsVnodeShmSize = TSDB_MAX_MSG_SIZE * 10 + 1024; int32_t tsQnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; int32_t tsSnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; -int32_t tsBnodeShmSize = TSDB_MAX_MSG_SIZE * 4 + 1024; int32_t tsNumOfShmThreads = 1; // queue & threads @@ -358,7 +357,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "vnodeShmSize", tsVnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "qnodeShmSize", tsQnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "snodeShmSize", tsSnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "bnodeShmSize", tsBnodeShmSize, TSDB_MAX_MSG_SIZE * 2 + 1024, INT32_MAX, 0) != 0) return -1; if (cfgAddInt32(pCfg, "numOfShmThreads", tsNumOfShmThreads, 1, 1024, 0) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; @@ -707,7 +705,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsVnodeShmSize = cfgGetItem(pCfg, "vnodeShmSize")->i32; tsQnodeShmSize = cfgGetItem(pCfg, "qnodeShmSize")->i32; tsSnodeShmSize = cfgGetItem(pCfg, "snodeShmSize")->i32; - tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; @@ -780,12 +777,6 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } break; } - case 'b': { - if (strcasecmp("bnodeShmSize", name) == 0) { - tsBnodeShmSize = cfgGetItem(pCfg, "bnodeShmSize")->i32; - } - break; - } case 'c': { if (strcasecmp("charset", name) == 0) { const char *locale = cfgGetItem(pCfg, "locale")->str; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 55e5616dd5..1f0023278f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1448,7 +1448,7 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { taosHashCleanup(pRsp->writeDbs); } -int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { +int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1461,7 +1461,7 @@ int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnod return tlen; } -int32_t tDeserializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { +int32_t tDeserializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index d9d4fb274e..bb82863e73 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -795,7 +795,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // not enough time range if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = taosTimeAdd(start, pInterval->interval, pInterval->intervalUnit, precision) - 1; - while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + while (end < t) { // move forward to the correct time window start += pInterval->sliding; if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index a01c393441..9660a37f48 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -366,6 +366,7 @@ static int32_t toBinary(SVariant *pVariant, char **pDest, int32_t *pDestSize) { } if (pBuf != NULL) { + taosMemoryFree(pVariant->pz); *pDest = pBuf; } @@ -688,7 +689,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_BIGINT: { if (convertToInteger(pVariant, &result, type, true, false, converted) < 0) { - SET_EXT_INFO(converted, (int64_t)result, INT64_MIN + 1, INT64_MAX, extInfo); + SET_EXT_INFO(converted, result, INT64_MIN + 1, INT64_MAX, extInfo); return -1; } *((int64_t *)payload) = (int64_t)result; @@ -697,7 +698,7 @@ int32_t tVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool inc case TSDB_DATA_TYPE_UBIGINT: { if (convertToInteger(pVariant, &result, type, false, false, converted) < 0) { - SET_EXT_INFO(converted, (uint64_t)result, 0, UINT64_MAX - 1, extInfo); + SET_EXT_INFO(converted, result, 0, UINT64_MAX - 1, extInfo); return -1; } *((uint64_t *)payload) = (uint64_t)result; diff --git a/source/dnode/CMakeLists.txt b/source/dnode/CMakeLists.txt index 87e4c5fc46..035cea2dc4 100644 --- a/source/dnode/CMakeLists.txt +++ b/source/dnode/CMakeLists.txt @@ -2,5 +2,4 @@ add_subdirectory(mnode) add_subdirectory(vnode) add_subdirectory(qnode) add_subdirectory(snode) -add_subdirectory(bnode) add_subdirectory(mgmt) \ No newline at end of file diff --git a/source/dnode/bnode/CMakeLists.txt b/source/dnode/bnode/CMakeLists.txt deleted file mode 100644 index 77f4d15ad2..0000000000 --- a/source/dnode/bnode/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -aux_source_directory(src BNODE_SRC) -add_library(bnode STATIC ${BNODE_SRC}) -target_include_directories( - bnode - PUBLIC "${TD_SOURCE_DIR}/include/dnode/bnode" - private "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - bnode - PRIVATE transport - PRIVATE os - PRIVATE common - PRIVATE util -) \ No newline at end of file diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h deleted file mode 100644 index e611d230a3..0000000000 --- a/source/dnode/bnode/inc/bndInt.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_BNODE_INT_H_ -#define _TD_BNODE_INT_H_ - -#include "os.h" - -#include "tarray.h" -#include "tlog.h" -#include "tmsg.h" -#include "trpc.h" - -#include "bnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBnode { - SMsgCb msgCb; -} SBnode; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c deleted file mode 100644 index b9c41ebf43..0000000000 --- a/source/dnode/bnode/src/bnode.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "bndInt.h" - -SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { - SBnode *pBnode = taosMemoryCalloc(1, sizeof(SBnode)); - pBnode->msgCb = pOption->msgCb; - return pBnode; -} - -void bndClose(SBnode *pBnode) { taosMemoryFree(pBnode); } - -int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; } - -int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; } diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 581686ba90..423028b167 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -1,6 +1,5 @@ add_subdirectory(node_mgmt) add_subdirectory(node_util) -add_subdirectory(mgmt_bnode) add_subdirectory(mgmt_mnode) add_subdirectory(mgmt_qnode) add_subdirectory(mgmt_snode) diff --git a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt b/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt deleted file mode 100644 index 0a6cf52fb8..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -aux_source_directory(src MGMT_BNODE) -add_library(mgmt_bnode STATIC ${MGMT_BNODE}) -target_include_directories( - mgmt_bnode - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - mgmt_bnode node_util -) \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h b/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h deleted file mode 100644 index c05ad46189..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/inc/bmInt.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_DND_BNODE_INT_H_ -#define _TD_DND_BNODE_INT_H_ - -#include "dmUtil.h" - -#include "bnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct SBnodeMgmt { - SDnodeData *pData; - SBnode *pBnode; - SMsgCb msgCb; - const char *path; - const char *name; - SMultiWorker writeWorker; - SSingleWorker monitorWorker; -} SBnodeMgmt; - -// bmHandle.c -SArray *bmGetMsgHandles(); -int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t bmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); - -// bmWorker.c -int32_t bmStartWorker(SBnodeMgmt *pMgmt); -void bmStopWorker(SBnodeMgmt *pMgmt); -int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_DND_BNODE_INT_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c b/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c deleted file mode 100644 index 9ec445c69c..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmHandle.c +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -void bmGetMonitorInfo(SBnodeMgmt *pMgmt, SMonBmInfo *bmInfo) {} - -int32_t bmProcessGetMonBmInfoReq(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SMonBmInfo bmInfo = {0}; - bmGetMonitorInfo(pMgmt, &bmInfo); - dmGetMonitorSystemInfo(&bmInfo.sys); - monGetLogs(&bmInfo.log); - - int32_t rspLen = tSerializeSMonBmInfo(NULL, 0, &bmInfo); - if (rspLen < 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - void *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tSerializeSMonBmInfo(pRsp, rspLen, &bmInfo); - pMsg->info.rsp = pRsp; - pMsg->info.rspLen = rspLen; - tFreeSMonBmInfo(&bmInfo); - return 0; -} - -int32_t bmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { - SDCreateBnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (pInput->pData->dnodeId != 0 && createReq.dnodeId != pInput->pData->dnodeId) { - terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pInput->pData->dnodeId); - return -1; - } - - bool deployed = true; - if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { - dError("failed to write bnode file since %s", terrstr()); - return -1; - } - - return 0; -} - -int32_t bmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { - SDDropBnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { - terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to drop bnode since %s", terrstr()); - return -1; - } - - bool deployed = false; - if (dmWriteFile(pInput->path, pInput->name, deployed) != 0) { - dError("failed to write bnode file since %s", terrstr()); - return -1; - } - - return 0; -} - -SArray *bmGetMsgHandles() { - int32_t code = -1; - SArray *pArray = taosArrayInit(2, sizeof(SMgmtHandle)); - if (pArray == NULL) goto _OVER; - - if (dmSetMgmtHandle(pArray, TDMT_MON_BM_INFO, bmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - - code = 0; -_OVER: - if (code != 0) { - taosArrayDestroy(pArray); - return NULL; - } else { - return pArray; - } -} diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c b/source/dnode/mgmt/mgmt_bnode/src/bmInt.c deleted file mode 100644 index 2c5d23cae9..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmInt.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -static int32_t bmRequire(const SMgmtInputOpt *pInput, bool *required) { - return dmReadFile(pInput->path, pInput->name, required); -} - -static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { pOption->msgCb = pMgmt->msgCb; } - -static void bmClose(SBnodeMgmt *pMgmt) { - if (pMgmt->pBnode != NULL) { - bmStopWorker(pMgmt); - bndClose(pMgmt->pBnode); - pMgmt->pBnode = NULL; - } - - taosMemoryFree(pMgmt); -} - -int32_t bmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { - SBnodeMgmt *pMgmt = taosMemoryCalloc(1, sizeof(SBnodeMgmt)); - if (pMgmt == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pMgmt->pData = pInput->pData; - pMgmt->path = pInput->path; - pMgmt->name = pInput->name; - pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.mgmt = pMgmt; - - SBnodeOpt option = {0}; - bmInitOption(pMgmt, &option); - pMgmt->pBnode = bndOpen(pMgmt->path, &option); - if (pMgmt->pBnode == NULL) { - dError("failed to open bnode since %s", terrstr()); - bmClose(pMgmt); - return -1; - } - tmsgReportStartup("bnode-impl", "initialized"); - - if (bmStartWorker(pMgmt) != 0) { - dError("failed to start bnode worker since %s", terrstr()); - bmClose(pMgmt); - return -1; - } - tmsgReportStartup("bnode-worker", "initialized"); - - pOutput->pMgmt = pMgmt; - return 0; -} - -SMgmtFunc bmGetMgmtFunc() { - SMgmtFunc mgmtFunc = {0}; - mgmtFunc.openFp = bmOpen; - mgmtFunc.closeFp = (NodeCloseFp)bmClose; - mgmtFunc.createFp = (NodeCreateFp)bmProcessCreateReq; - mgmtFunc.dropFp = (NodeDropFp)bmProcessDropReq; - mgmtFunc.requiredFp = bmRequire; - mgmtFunc.getHandlesFp = bmGetMsgHandles; - - return mgmtFunc; -} diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c deleted file mode 100644 index 08c9edd854..0000000000 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "bmInt.h" - -static void bmSendErrorRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); - - dTrace("msg:%p, is freed", pMsg); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - if (pMsg != NULL) { - bmSendErrorRsp(pMsg, code); - } - } -} - -static inline void bmSendRsp(SRpcMsg *pMsg, int32_t code) { - SRpcMsg rsp = { - .code = code, - .pCont = pMsg->info.rsp, - .contLen = pMsg->info.rspLen, - .info = pMsg->info, - }; - tmsgSendRsp(&rsp); -} - -static void bmProcessMonitorQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SBnodeMgmt *pMgmt = pInfo->ahandle; - int32_t code = -1; - dTrace("msg:%p, get from bnode-monitor queue", pMsg); - - if (pMsg->msgType == TDMT_MON_BM_INFO) { - code = bmProcessGetMonBmInfoReq(pMgmt, pMsg); - } else { - terrno = TSDB_CODE_MSG_NOT_PROCESSED; - } - - if (IsReq(pMsg)) { - if (code != 0 && terrno != 0) code = terrno; - bmSendRsp(pMsg, code); - } - - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - -static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { - SBnodeMgmt *pMgmt = pInfo->ahandle; - - SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); - if (pArray == NULL) { - bmSendErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY); - return; - } - - for (int32_t i = 0; i < numOfMsgs; ++i) { - SRpcMsg *pMsg = NULL; - taosGetQitem(qall, (void **)&pMsg); - if (pMsg != NULL) { - dTrace("msg:%p, get from bnode-write queue", pMsg); - if (taosArrayPush(pArray, &pMsg) == NULL) { - bmSendErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY); - } - } - } - - bndProcessWMsgs(pMgmt->pBnode, pArray); - - for (size_t i = 0; i < numOfMsgs; i++) { - SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i); - if (pMsg != NULL) { - dTrace("msg:%p, is freed", pMsg); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } - } - taosArrayDestroy(pArray); -} - -int32_t bmPutNodeMsgToWriteQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SMultiWorker *pWorker = &pMgmt->writeWorker; - - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} - -int32_t bmPutNodeMsgToMonitorQueue(SBnodeMgmt *pMgmt, SRpcMsg *pMsg) { - SSingleWorker *pWorker = &pMgmt->monitorWorker; - - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); - taosWriteQitem(pWorker->queue, pMsg); - return 0; -} - -int32_t bmStartWorker(SBnodeMgmt *pMgmt) { - SMultiWorkerCfg cfg = { - .max = 1, - .name = "bnode-write", - .fp = (FItems)bmProcessWriteQueue, - .param = pMgmt, - }; - if (tMultiWorkerInit(&pMgmt->writeWorker, &cfg) != 0) { - dError("failed to start bnode-write worker since %s", terrstr()); - return -1; - } - - SSingleWorkerCfg mCfg = { - .min = 1, - .max = 1, - .name = "bnode-monitor", - .fp = (FItem)bmProcessMonitorQueue, - .param = pMgmt, - }; - if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { - dError("failed to start bnode-monitor worker since %s", terrstr()); - return -1; - } - - dDebug("bnode workers are initialized"); - return 0; -} - -void bmStopWorker(SBnodeMgmt *pMgmt) { - tSingleWorkerCleanup(&pMgmt->monitorWorker); - tMultiWorkerCleanup(&pMgmt->writeWorker); - dDebug("bnode workers are closed"); -} diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index dd98816161..782c828bb2 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -326,8 +326,6 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index e7ccce1c8b..ca8a434d05 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -132,12 +132,6 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_DROP_SNODE: code = (*pMgmt->processDropNodeFp)(SNODE, pMsg); break; - case TDMT_DND_CREATE_BNODE: - code = (*pMgmt->processCreateNodeFp)(BNODE, pMsg); - break; - case TDMT_DND_DROP_BNODE: - code = (*pMgmt->processDropNodeFp)(BNODE, pMsg); - break; case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index fc45dbf15f..105b6e542e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -105,7 +105,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; SDDropMnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -136,8 +136,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -163,8 +161,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c index 6382152cd1..0573be90ea 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmHandle.c @@ -56,7 +56,7 @@ int32_t qmProcessGetMonitorInfoReq(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDCreateQnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -78,7 +78,7 @@ int32_t qmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t qmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDDropQnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 7cb41ca77c..c6e3b3611f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -45,7 +45,7 @@ int32_t smProcessGetMonitorInfoReq(SSnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDCreateSnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -67,7 +67,7 @@ int32_t smProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t smProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SDDropSnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index 98027d80d4..f1be20289a 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -1,7 +1,7 @@ aux_source_directory(src IMPLEMENT_SRC) add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( - dnode mgmt_bnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode + dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode ) target_include_directories( dnode diff --git a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h index d3f1044f88..5f36309038 100644 --- a/source/dnode/mgmt/node_mgmt/inc/dmNodes.h +++ b/source/dnode/mgmt/node_mgmt/inc/dmNodes.h @@ -23,7 +23,6 @@ extern "C" { #endif SMgmtFunc dmGetMgmtFunc(); -SMgmtFunc bmGetMgmtFunc(); SMgmtFunc qmGetMgmtFunc(); SMgmtFunc smGetMgmtFunc(); SMgmtFunc vmGetMgmtFunc(); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d07ec7abb0..5a7f149bc6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,28 +51,34 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); + // sufficiency + if (!osDataSpaceSufficient()) { + dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); + } + if (!osLogSpaceSufficient()) { + dWarn("free log disk size: %f GB, not sufficient, expected %f GB at least", (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); + } + if (!osTempSpaceSufficient()) { + dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); + } + // availability + bool ret = true; if (!osDataSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least , quit", - (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + dError("data disk space unavailable, i.e. %s", tsDataDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; + ret = false; } if (!osLogSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsLogSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsLogSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + dError("log disk space unavailable, i.e. %s", tsLogDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; + ret = false; } if (!osTempSpaceAvailable()) { - dError("free disk size: %f GB, too little, require %f GB at least at least, quit", - (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, - (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); - terrno = TSDB_CODE_NO_AVAIL_DISK; - return false; + dError("temp disk space unavailable, i.e. %s", tsTempDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; + ret = false; } - return true; + return ret; } static bool dmCheckDataDirVersion() { diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2e149be3b6..75c76a1f54 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -141,7 +141,6 @@ int32_t dmInitDnode(SDnode *pDnode, EDndNodeType rtype) { pDnode->wrappers[VNODE].func = vmGetMgmtFunc(); pDnode->wrappers[QNODE].func = qmGetMgmtFunc(); pDnode->wrappers[SNODE].func = smGetMgmtFunc(); - pDnode->wrappers[BNODE].func = bmGetMgmtFunc(); for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; diff --git a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c index 1c71fe5e1f..be1de02e6a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMonitor.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMonitor.c @@ -41,7 +41,6 @@ static void dmGetMonitorDnodeInfo(SDnode *pDnode, SMonDnodeInfo *pInfo) { pInfo->has_mnode = pDnode->wrappers[MNODE].required; pInfo->has_qnode = pDnode->wrappers[QNODE].required; pInfo->has_snode = pDnode->wrappers[SNODE].required; - pInfo->has_bnode = pDnode->wrappers[BNODE].required; tstrncpy(pInfo->logdir.name, tsLogDir, sizeof(pInfo->logdir.name)); pInfo->logdir.size = tsLogSpace.size; tstrncpy(pInfo->tempdir.name, tsTempDir, sizeof(pInfo->tempdir.name)); @@ -116,21 +115,6 @@ static void dmGetSmMonitorInfo(SDnode *pDnode) { } } -static void dmGetBmMonitorInfo(SDnode *pDnode) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[BNODE]; - if (dmMarkWrapper(pWrapper) == 0) { - SMonBmInfo bmInfo = {0}; - if (tsMultiProcess) { - dmSendLocalRecv(pDnode, TDMT_MON_BM_INFO, tDeserializeSMonBmInfo, &bmInfo); - } else if (pWrapper->pMgmt != NULL) { - bmGetMonitorInfo(pWrapper->pMgmt, &bmInfo); - } - dmReleaseWrapper(pWrapper); - monSetBmInfo(&bmInfo); - tFreeSMonBmInfo(&bmInfo); - } -} - void dmSendMonitorReport() { if (!tsEnableMonitor || tsMonitorFqdn[0] == 0 || tsMonitorPort == 0) return; dTrace("send monitor report to %s:%u", tsMonitorFqdn, tsMonitorPort); @@ -141,7 +125,6 @@ void dmSendMonitorReport() { dmGetVmMonitorInfo(pDnode); dmGetQmMonitorInfo(pDnode); dmGetSmMonitorInfo(pDnode); - dmGetBmMonitorInfo(pDnode); monSendReport(); } diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 99ffd73a7a..ec5f3589e0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -26,8 +26,6 @@ static int32_t dmCreateShm(SMgmtWrapper *pWrapper) { shmsize = tsSnodeShmSize; } else if (pWrapper->ntype == MNODE) { shmsize = tsMnodeShmSize; - } else if (pWrapper->ntype == BNODE) { - shmsize = tsBnodeShmSize; } else { return -1; } diff --git a/source/dnode/mgmt/node_util/CMakeLists.txt b/source/dnode/mgmt/node_util/CMakeLists.txt index 5d879fdbcf..5c670cbdd3 100644 --- a/source/dnode/mgmt/node_util/CMakeLists.txt +++ b/source/dnode/mgmt/node_util/CMakeLists.txt @@ -6,5 +6,5 @@ target_include_directories( PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - node_util cjson mnode vnode qnode snode bnode wal sync taos_static tfs monitor + node_util cjson mnode vnode qnode snode wal sync taos_static tfs monitor ) \ No newline at end of file diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index f599de384c..2f9c0aff2f 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -68,8 +68,7 @@ typedef enum { VNODE = 2, QNODE = 3, SNODE = 4, - BNODE = 5, - NODE_END = 6, + NODE_END = 5, } EDndNodeType; typedef enum { diff --git a/source/dnode/mgmt/node_util/src/dmUtil.c b/source/dnode/mgmt/node_util/src/dmUtil.c index 832e15a1e0..f5364b3195 100644 --- a/source/dnode/mgmt/node_util/src/dmUtil.c +++ b/source/dnode/mgmt/node_util/src/dmUtil.c @@ -39,8 +39,6 @@ const char *dmNodeLogName(EDndNodeType ntype) { return "snode"; case MNODE: return "mnode"; - case BNODE: - return "bnode"; default: return "taosd"; } @@ -56,8 +54,6 @@ const char *dmNodeProcName(EDndNodeType ntype) { return "taoss"; case MNODE: return "taosm"; - case BNODE: - return "taosb"; default: return "taosd"; } @@ -73,8 +69,6 @@ const char *dmNodeName(EDndNodeType ntype) { return "snode"; case MNODE: return "mnode"; - case BNODE: - return "bnode"; default: return "dnode"; } diff --git a/source/dnode/mgmt/test/CMakeLists.txt b/source/dnode/mgmt/test/CMakeLists.txt index 6b1919bf18..6709a2aa5e 100644 --- a/source/dnode/mgmt/test/CMakeLists.txt +++ b/source/dnode/mgmt/test/CMakeLists.txt @@ -1,7 +1,6 @@ if(${BUILD_TEST}) enable_testing() add_subdirectory(qnode) - add_subdirectory(bnode) add_subdirectory(snode) #add_subdirectory(mnode) add_subdirectory(vnode) diff --git a/source/dnode/mgmt/test/bnode/CMakeLists.txt b/source/dnode/mgmt/test/bnode/CMakeLists.txt deleted file mode 100644 index 7108d3adb9..0000000000 --- a/source/dnode/mgmt/test/bnode/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -aux_source_directory(. DND_BNODE_TEST_SRC) -add_executable(dbnodeTest ${DND_BNODE_TEST_SRC}) -target_link_libraries( - dbnodeTest sut -) - -add_test( - NAME dbnodeTest - COMMAND dbnodeTest -) diff --git a/source/dnode/mgmt/test/bnode/dbnode.cpp b/source/dnode/mgmt/test/bnode/dbnode.cpp deleted file mode 100644 index 24d8c594ce..0000000000 --- a/source/dnode/mgmt/test/bnode/dbnode.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/** - * @file dbnode.cpp - * @author slguan (slguan@taosdata.com) - * @brief DNODE module bnode tests - * @version 1.0 - * @date 2022-01-05 - * - * @copyright Copyright (c) 2022 - * - */ - -#include "sut.h" - -class DndTestBnode : public ::testing::Test { - protected: - static void SetUpTestSuite() { - test.Init(TD_TMP_DIR_PATH "dbnodeTest", 9112); - taosMsleep(1100); - } - static void TearDownTestSuite() { test.Cleanup(); } - static Testbase test; - - public: - void SetUp() override {} - void TearDown() override {} -}; - -Testbase DndTestBnode::test; - -TEST_F(DndTestBnode, 01_Create_Bnode) { - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); - } - - // test.Restart(); - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); - } -} - -TEST_F(DndTestBnode, 02_Drop_Bnode) { - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION); - } - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); - } - - // test.Restart(); - - { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); - } - - { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - } -} diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 857f58befc..7b7eb97216 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -139,9 +139,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -152,9 +152,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -165,9 +165,9 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SDDropMnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index ef51be47a6..a2c6a2c28c 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -30,9 +30,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -43,9 +43,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -56,9 +56,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -71,9 +71,9 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -87,9 +87,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -101,9 +101,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -114,9 +114,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SDCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index 9ae0fbdc54..e3ad65d831 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -30,9 +30,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -43,9 +43,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -56,9 +56,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -71,9 +71,9 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -87,9 +87,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -101,9 +101,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -114,9 +114,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SDCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 699203e8c1..a4d2e46881 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -43,9 +43,7 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { -#ifdef _TD_DARWIN_64 osDefaultInit(); -#endif tsServerPort = port; strcpy(tsLocalFqdn, "localhost"); snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); diff --git a/source/dnode/mnode/impl/inc/mndBnode.h b/source/dnode/mnode/impl/inc/mndBnode.h deleted file mode 100644 index cefc0b1859..0000000000 --- a/source/dnode/mnode/impl/inc/mndBnode.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_MND_BNODE_H_ -#define _TD_MND_BNODE_H_ - -#include "mndInt.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int32_t mndInitBnode(SMnode *pMnode); -void mndCleanupBnode(SMnode *pMnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_MND_BNODE_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9cc920de04..48eea7daea 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -43,8 +43,6 @@ typedef enum { MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, - MND_OPER_CREATE_BNODE, - MND_OPER_DROP_BNODE, MND_OPER_CREATE_DNODE, MND_OPER_DROP_DNODE, MND_OPER_CONFIG_DNODE, @@ -226,13 +224,6 @@ typedef struct { SDnodeObj* pDnode; } SSnodeObj; -typedef struct { - int32_t id; - int64_t createdTime; - int64_t updateTime; - SDnodeObj* pDnode; -} SBnodeObj; - typedef struct { int32_t maxUsers; int32_t maxDbs; diff --git a/source/dnode/mnode/impl/inc/mndQuery.h b/source/dnode/mnode/impl/inc/mndQuery.h index 4b359c0a14..69ec54f330 100644 --- a/source/dnode/mnode/impl/inc/mndQuery.h +++ b/source/dnode/mnode/impl/inc/mndQuery.h @@ -22,6 +22,8 @@ extern "C" { #endif +#define MAX_META_MSG_IN_BATCH 1048576 + int32_t mndInitQuery(SMnode *pMnode); void mndCleanupQuery(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c deleted file mode 100644 index 06209d89c7..0000000000 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ /dev/null @@ -1,447 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "mndBnode.h" -#include "mndDnode.h" -#include "mndPrivilege.h" -#include "mndShow.h" -#include "mndTrans.h" -#include "mndUser.h" - -#define BNODE_VER_NUMBER 1 -#define BNODE_RESERVE_SIZE 64 - -static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj); -static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw); -static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj); -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew); -static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj); -static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq); -static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq); -static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); -static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter); - -int32_t mndInitBnode(SMnode *pMnode) { - SSdbTable table = { - .sdbType = SDB_BNODE, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndBnodeActionEncode, - .decodeFp = (SdbDecodeFp)mndBnodeActionDecode, - .insertFp = (SdbInsertFp)mndBnodeActionInsert, - .updateFp = (SdbUpdateFp)mndBnodeActionUpdate, - .deleteFp = (SdbDeleteFp)mndBnodeActionDelete, - }; - - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_BNODE, mndProcessCreateBnodeReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_BNODE, mndProcessDropBnodeReq); - mndSetMsgHandle(pMnode, TDMT_DND_CREATE_BNODE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_DND_DROP_BNODE_RSP, mndTransProcessRsp); - - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndRetrieveBnodes); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_BNODE, mndCancelGetNextBnode); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupBnode(SMnode *pMnode) {} - -static SBnodeObj *mndAcquireBnode(SMnode *pMnode, int32_t bnodeId) { - SBnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_BNODE, &bnodeId); - if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { - terrno = TSDB_CODE_MND_BNODE_NOT_EXIST; - } - return pObj; -} - -static void mndReleaseBnode(SMnode *pMnode, SBnodeObj *pObj) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pObj); -} - -static SSdbRaw *mndBnodeActionEncode(SBnodeObj *pObj) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - - SSdbRaw *pRaw = sdbAllocRaw(SDB_BNODE, BNODE_VER_NUMBER, sizeof(SBnodeObj) + BNODE_RESERVE_SIZE); - if (pRaw == NULL) goto _OVER; - - int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER) - SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER) - SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER) - SDB_SET_RESERVE(pRaw, dataPos, BNODE_RESERVE_SIZE, _OVER) - - terrno = 0; - -_OVER: - if (terrno != 0) { - mError("bnode:%d, failed to encode to raw:%p since %s", pObj->id, pRaw, terrstr()); - sdbFreeRaw(pRaw); - return NULL; - } - - mTrace("bnode:%d, encode to raw:%p, row:%p", pObj->id, pRaw, pObj); - return pRaw; -} - -static SSdbRow *mndBnodeActionDecode(SSdbRaw *pRaw) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - - int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - - if (sver != BNODE_VER_NUMBER) { - terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto _OVER; - } - - SSdbRow *pRow = sdbAllocRow(sizeof(SBnodeObj)); - if (pRow == NULL) goto _OVER; - - SBnodeObj *pObj = sdbGetRowObj(pRow); - if (pObj == NULL) goto _OVER; - - int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER) - SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER) - SDB_GET_RESERVE(pRaw, dataPos, BNODE_RESERVE_SIZE, _OVER) - - terrno = 0; - -_OVER: - if (terrno != 0) { - mError("bnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr()); - taosMemoryFreeClear(pRow); - return NULL; - } - - mTrace("bnode:%d, decode from raw:%p, row:%p", pObj->id, pRaw, pObj); - return pRow; -} - -static int32_t mndBnodeActionInsert(SSdb *pSdb, SBnodeObj *pObj) { - mTrace("bnode:%d, perform insert action, row:%p", pObj->id, pObj); - pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id); - if (pObj->pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - mError("bnode:%d, failed to perform insert action since %s", pObj->id, terrstr()); - return -1; - } - - return 0; -} - -static int32_t mndBnodeActionDelete(SSdb *pSdb, SBnodeObj *pObj) { - mTrace("bnode:%d, perform delete action, row:%p", pObj->id, pObj); - if (pObj->pDnode != NULL) { - sdbRelease(pSdb, pObj->pDnode); - pObj->pDnode = NULL; - } - - return 0; -} - -static int32_t mndBnodeActionUpdate(SSdb *pSdb, SBnodeObj *pOld, SBnodeObj *pNew) { - mTrace("bnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); - pOld->updateTime = pNew->updateTime; - return 0; -} - -static int32_t mndSetCreateBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeUndoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pUndoRaw = mndBnodeActionEncode(pObj); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; - return 0; -} - -static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDCreateBnodeReq createReq = {0}; - createReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_CREATE_BNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (mndTransAppendUndoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndCreateBnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateBnodeReq *pCreate) { - int32_t code = -1; - - SBnodeObj bnodeObj = {0}; - bnodeObj.id = pDnode->id; - bnodeObj.createdTime = taosGetTimestampMs(); - bnodeObj.updateTime = bnodeObj.createdTime; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-bnode"); - if (pTrans == NULL) goto _OVER; - - mInfo("trans:%d, used to create bnode:%d", pTrans->id, pCreate->dnodeId); - if (mndSetCreateBnodeRedoLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeUndoLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeCommitLogs(pTrans, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeRedoActions(pTrans, pDnode, &bnodeObj) != 0) goto _OVER; - if (mndSetCreateBnodeUndoActions(pTrans, pDnode, &bnodeObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - - code = 0; - -_OVER: - mndTransDrop(pTrans); - return code; -} - -static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SBnodeObj *pObj = NULL; - SDnodeObj *pDnode = NULL; - SMCreateBnodeReq createReq = {0}; - - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mInfo("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { - goto _OVER; - } - - pObj = mndAcquireBnode(pMnode, createReq.dnodeId); - if (pObj != NULL) { - terrno = TSDB_CODE_MND_BNODE_ALREADY_EXIST; - goto _OVER; - } else if (terrno != TSDB_CODE_MND_BNODE_NOT_EXIST) { - goto _OVER; - } - - pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); - if (pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; - } - - code = mndCreateBnode(pMnode, pReq, pDnode, &createReq); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("bnode:%d, failed to create since %s", createReq.dnodeId, terrstr()); - } - - mndReleaseBnode(pMnode, pObj); - mndReleaseDnode(pMnode, pDnode); - return code; -} - -static int32_t mndSetDropBnodeRedoLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pRedoRaw = mndBnodeActionEncode(pObj); - if (pRedoRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; - return 0; -} - -static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { - SSdbRaw *pCommitRaw = mndBnodeActionEncode(pObj); - if (pCommitRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; - return 0; -} - -static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq dropReq = {0}; - dropReq.dnodeId = pDnode->id; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - STransAction action = {0}; - action.epSet = mndGetDnodeEpset(pDnode); - action.pCont = pReq; - action.contLen = contLen; - action.msgType = TDMT_DND_DROP_BNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; - - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(pReq); - return -1; - } - - return 0; -} - -static int32_t mndDropBnode(SMnode *pMnode, SRpcMsg *pReq, SBnodeObj *pObj) { - int32_t code = -1; - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "drop-bnode"); - if (pTrans == NULL) goto _OVER; - - mInfo("trans:%d, used to drop bnode:%d", pTrans->id, pObj->id); - if (mndSetDropBnodeRedoLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropBnodeCommitLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropBnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - - code = 0; - -_OVER: - mndTransDrop(pTrans); - return code; -} - -static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SBnodeObj *pObj = NULL; - SMDropBnodeReq dropReq = {0}; - - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mInfo("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { - goto _OVER; - } - - if (dropReq.dnodeId <= 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - pObj = mndAcquireBnode(pMnode, dropReq.dnodeId); - if (pObj == NULL) { - goto _OVER; - } - - code = mndDropBnode(pMnode, pReq, pObj); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; - -_OVER: - if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("bnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr()); - } - - mndReleaseBnode(pMnode, pObj); - return code; -} - -static int32_t mndRetrieveBnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SBnodeObj *pObj = NULL; - - while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_BNODE, pShow->pIter, (void **)&pObj); - if (pShow->pIter == NULL) break; - - cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); - - char buf[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(buf, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, buf, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); - - numOfRows++; - sdbRelease(pSdb, pObj); - } - - pShow->numOfRows += numOfRows; - - return numOfRows; -} - -static void mndCancelGetNextBnode(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index c1c7581a7e..7d0f5742f8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -447,10 +447,6 @@ void dumpDnode(SSdb *pSdb, SJson *json) { } } -void dumpBnode(SSdb *pSdb, SJson *json) { - // not implemented yet -} - void dumpSnode(SSdb *pSdb, SJson *json) { void *pIter = NULL; SJson *items = tjsonAddArrayToObject(json, "snodes"); @@ -616,7 +612,6 @@ void mndDumpSdb() { dumpAuth(pSdb, json); dumpUser(pSdb, json); dumpDnode(pSdb, json); - dumpBnode(pSdb, json); dumpSnode(pSdb, json); dumpQnode(pSdb, json); dumpMnode(pSdb, json); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a52ca1bd42..4eb2e2978b 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,6 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" #include "mndDb.h" @@ -232,7 +231,7 @@ static int32_t mndInitWal(SMnode *pMnode) { pMnode->pWal = walOpen(path, &cfg); if (pMnode->pWal == NULL) { - mError("failed to open wal since %s", terrstr()); + mError("failed to open wal since %s. wal:%s", terrstr(), path); return -1; } @@ -296,7 +295,6 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-mnode", mndInitMnode, mndCleanupMnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-qnode", mndInitQnode, mndCleanupQnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-snode", mndInitSnode, mndCleanupSnode) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-bnode", mndInitBnode, mndCleanupBnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; @@ -603,22 +601,27 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { return 0; } if (mndAcquireRpc(pMsg->info.node) == 0) return 0; + + SMnode *pMnode = pMsg->info.node; + const char *role = syncGetMyRoleStr(pMnode->syncMgmt.sync); + bool restored = syncIsRestoreFinish(pMnode->syncMgmt.sync); if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { + mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, + pMnode->stopped, restored, role); return -1; } - SEpSet epSet = {0}; - SMnode *pMnode = pMsg->info.node; + const STraceId *trace = &pMsg->info.traceId; + SEpSet epSet = {0}; mndGetMnodeEpSet(pMnode, &epSet); - const STraceId *trace = &pMsg->info.traceId; mDebug( - "msg:%p, failed to check mnode state since %s, mnode restored:%d stopped:%d, sync restored:%d role:%s type:%s " - "numOfEps:%d inUse:%d", - pMsg, terrstr(), pMnode->restored, pMnode->stopped, syncIsRestoreFinish(pMnode->syncMgmt.sync), - syncGetMyRoleStr(pMnode->syncMgmt.sync), TMSG_INFO(pMsg->msgType), epSet.numOfEps, epSet.inUse); + "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " + "role:%s, redirect numOfEps:%d inUse:%d", + pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, restored, role, epSet.numOfEps, + epSet.inUse); if (epSet.numOfEps > 0) { for (int32_t i = 0; i < epSet.numOfEps; ++i) { diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c27241317c..f6ea81c565 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -324,9 +324,9 @@ static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pA } static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDropReq, SEpSet *pDroprEpSet) { - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, pDropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, pDropReq); void *pReq = taosMemoryMalloc(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, pDropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, pDropReq); STransAction action = { .epSet = *pDroprEpSet, @@ -394,7 +394,6 @@ static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndTransAppendNullLog(pTrans) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -411,7 +410,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateMnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -478,7 +477,6 @@ static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeO static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; - int32_t numOfReplicas = 0; SDDropMnodeReq dropReq = {0}; SEpSet dropEpSet = {0}; @@ -505,9 +503,8 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) { if (pObj == NULL) return 0; - if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) return -1; - if (mndTransAppendNullLog(pTrans) != 0) return -1; + if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; return 0; } @@ -536,7 +533,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { SMnodeObj *pObj = NULL; SMDropMnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -715,7 +712,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) { SMnodeObj *pObj = NULL; ESdbStatus objStatus = 0; void *pIter = NULL; - bool hasUpdatingMnode = false; + int32_t updatingMnodes = 0; + int32_t readyMnodes = 0; SSyncCfg cfg = {.myIndex = -1}; while (1) { @@ -723,7 +721,11 @@ static void mndReloadSyncConfig(SMnode *pMnode) { if (pIter == NULL) break; if (objStatus == SDB_STATUS_CREATING || objStatus == SDB_STATUS_DROPPING) { mInfo("vgId:1, has updating mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); - hasUpdatingMnode = true; + updatingMnodes++; + } + if (objStatus == SDB_STATUS_READY) { + mInfo("vgId:1, has ready mnode:%d, status:%s", pObj->id, sdbStatusName(objStatus)); + readyMnodes++; } if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { @@ -739,18 +741,25 @@ static void mndReloadSyncConfig(SMnode *pMnode) { sdbReleaseLock(pSdb, pObj, false); } + if (readyMnodes <= 0 || updatingMnodes <= 0) { + mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); + return; + } + // ASSERT(0); + if (cfg.myIndex == -1) { - mInfo("vgId:1, mnode not reload since selfIndex is -1"); +#if 1 + mInfo("vgId:1, mnode sync not reconfig since selfIndex is -1"); +#else + // cannot reconfig because the leader may fail to elect after reboot + mInfo("vgId:1, mnode sync not reconfig since selfIndex is -1, do sync stop oper"); + syncStop(pMnode->syncMgmt.sync); +#endif return; } - if (!mndGetRestored(pMnode)) { - mInfo("vgId:1, mnode not reload since restore not finished"); - return; - } - - if (hasUpdatingMnode) { - mInfo("vgId:1, start to reload mnode sync, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); + if (updatingMnodes > 0) { + mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); for (int32_t i = 0; i < cfg.replicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 1909b168bd..d42b66455f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -369,7 +369,7 @@ static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) { taosCacheRelease(pMgmt->appCache, (void **)&pApp, false); } -void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { +SAppObj *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { SAppObj *pApp = NULL; bool hasNext = taosCacheIterNext(pIter); if (hasNext) { diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index d48a446e20..70f9b35312 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -190,13 +190,13 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S SDCreateQnodeReq createReq = {0}; createReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -217,13 +217,13 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -273,7 +273,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateQnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -330,13 +330,13 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn SDDropQnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -384,7 +384,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { SQnodeObj *pObj = NULL; SMDropQnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index f185cdeaec..3e4c8005b4 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -77,6 +77,12 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { void *pRsp = NULL; SMnode *pMnode = pMsg->info.node; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + mError("too many msgs %d in mnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -84,14 +90,39 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgIdx); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msgLen = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgLen); + if (offset >= pMsg->contLen) { + mError("offset %d is bigger than contLen %d", offset, pMsg->contLen); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); + return -1; + } req.msg = (char *)pMsg->pCont + offset; offset += req.msgLen; @@ -106,6 +137,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { if (fp == NULL) { mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_MSG_NOT_PROCESSED; + taosArrayDestroy(batchRsp); return -1; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 511a84290d..1c8ec0a302 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -416,7 +416,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } SStreamTask* pTask = tNewSStreamTask(pStream->uid); - if (pInnerTask == NULL) { + if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 5a998dfe98..8b4cacf8d6 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -60,8 +60,6 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_MODULE; } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) { type = TSDB_MGMT_TABLE_QNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) { - type = TSDB_MGMT_TABLE_BNODE; } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) { type = TSDB_MGMT_TABLE_SNODE; } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) { diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 916e8d0c44..8127d5912e 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -195,13 +195,13 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S SDCreateSnodeReq createReq = {0}; createReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -222,13 +222,13 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -282,7 +282,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SMCreateSnodeReq createReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } @@ -341,13 +341,13 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn SDDropSnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void *pReq = taosMemoryMalloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); @@ -398,7 +398,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { SSnodeObj *pObj = NULL; SMDropSnodeReq dropReq = {0}; - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + if (tDeserializeSCreateDropMQSNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 10cfb0a660..8a3179b2a9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2553,12 +2553,17 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); + char *sep = ", "; + int32_t sepLen = strlen(sep); + int32_t rollupLen = sizeof(rollup) - 2; for (int32_t i = 0; i < rollupNum; ++i) { char *funcName = taosArrayGet(pStb->pFuncs, i); if (i) { - strcat(varDataVal(rollup), ", "); + strncat(varDataVal(rollup), sep, rollupLen); + rollupLen -= sepLen; } - strcat(varDataVal(rollup), funcName); + strncat(varDataVal(rollup), funcName, rollupLen); + rollupLen -= strlen(funcName); } varDataSetLen(rollup, strlen(varDataVal(rollup))); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 866a3fa8b9..ac95dd2795 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -293,6 +293,14 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (code == 0) { tsem_wait(&pMgmt->syncSem); + } else if (code > 0) { + mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); + taosWLockLatch(&pMgmt->lock); + pMgmt->transId = 0; + taosWUnLockLatch(&pMgmt->lock); + sdbWriteWithoutFree(pMnode->pSdb, pRaw); + sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID); + code = 0; } else if (code == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { terrno = TSDB_CODE_APP_NOT_READY; } else if (code == -1 && terrno == TSDB_CODE_SYN_INTERNAL_ERROR) { diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 3b1ca0999c..7db8485561 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -1,7 +1,6 @@ enable_testing() add_subdirectory(acct) -add_subdirectory(bnode) add_subdirectory(db) #add_subdirectory(dnode) add_subdirectory(func) diff --git a/source/dnode/mnode/impl/test/bnode/CMakeLists.txt b/source/dnode/mnode/impl/test/bnode/CMakeLists.txt deleted file mode 100644 index 2dd7b9ef78..0000000000 --- a/source/dnode/mnode/impl/test/bnode/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# aux_source_directory(. MNODE_BNODE_TEST_SRC) -# add_executable(mbnodeTest ${MNODE_BNODE_TEST_SRC}) -# target_link_libraries( -# mbnodeTest -# PUBLIC sut -# ) - -# add_test( -# NAME mbnodeTest -# COMMAND mbnodeTest -# ) diff --git a/source/dnode/mnode/impl/test/bnode/mbnode.cpp b/source/dnode/mnode/impl/test/bnode/mbnode.cpp deleted file mode 100644 index c93e2142d0..0000000000 --- a/source/dnode/mnode/impl/test/bnode/mbnode.cpp +++ /dev/null @@ -1,293 +0,0 @@ -/** - * @file bnode.cpp - * @author slguan (slguan@taosdata.com) - * @brief MNODE module bnode tests - * @version 1.0 - * @date 2022-01-05 - * - * @copyright Copyright (c) 2022 - * - */ - -#include "sut.h" - -class MndTestBnode : public ::testing::Test { - public: - void SetUp() override {} - void TearDown() override {} - - public: - static void SetUpTestSuite() { - test.Init(TD_TMP_DIR_PATH "mnode_test_bnode1", 9018); - const char* fqdn = "localhost"; - const char* firstEp = "localhost:9018"; - - server2.Start(TD_TMP_DIR_PATH "mnode_test_bnode2", 9019); - taosMsleep(300); - } - - static void TearDownTestSuite() { - server2.Stop(); - test.Cleanup(); - } - - static Testbase test; - static TestServer server2; -}; - -Testbase MndTestBnode::test; -TestServer MndTestBnode::server2; - -TEST_F(MndTestBnode, 01_Show_Bnode) { - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 0); -} - -TEST_F(MndTestBnode, 02_Create_Bnode) { - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DNODE_NOT_EXIST); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 1); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 1; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_ALREADY_EXIST); - } -} - -TEST_F(MndTestBnode, 03_Drop_Bnode) { - { - SCreateDnodeReq createReq = {0}; - strcpy(createReq.fqdn, "localhost"); - createReq.port = 9019; - - int32_t contLen = tSerializeSCreateDnodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDnodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - taosMsleep(1300); - test.SendShowReq(TSDB_MGMT_TABLE_DNODE, "dnodes", ""); - EXPECT_EQ(test.GetShowRows(), 2); - } - - { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 2); - } - - { - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); - - test.SendShowReq(TSDB_MGMT_TABLE_BNODE, "bnodes", ""); - EXPECT_EQ(test.GetShowRows(), 1); - } - - { - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_BNODE_NOT_EXIST); - } -} - -TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { - { - // send message first, then dnode2 crash, result is returned, and rollback is started - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - server2.Stop(); - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); - } - - { - // continue send message, bnode is creating - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); - } - - { - // continue send message, bnode is creating - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); - } - - { - // server start, wait until the rollback finished - server2.DoStart(); - taosMsleep(1000); - - int32_t retry = 0; - int32_t retryMax = 20; - - for (retry = 0; retry < retryMax; retry++) { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (pRsp->code == 0) break; - taosMsleep(1000); - } - - ASSERT_NE(retry, retryMax); - } -} - -TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { - { - // send message first, then dnode2 crash, result is returned, and rollback is started - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - server2.Stop(); - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); - } - - { - // continue send message, bnode is dropping - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); - } - - { - // continue send message, bnode is dropping - SMDropBnodeReq dropReq = {0}; - dropReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); - } - - { - // server start, wait until the rollback finished - server2.DoStart(); - taosMsleep(1000); - - int32_t retry = 0; - int32_t retryMax = 20; - - for (retry = 0; retry < retryMax; retry++) { - SMCreateBnodeReq createReq = {0}; - createReq.dnodeId = 2; - - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); - void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); - - SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); - ASSERT_NE(pRsp, nullptr); - if (pRsp->code == 0) break; - taosMsleep(1000); - } - - ASSERT_NE(retry, retryMax); - } -} diff --git a/source/dnode/mnode/impl/test/mnode/mnode.cpp b/source/dnode/mnode/impl/test/mnode/mnode.cpp index 1ed613c723..1f6dbd6dca 100644 --- a/source/dnode/mnode/impl/test/mnode/mnode.cpp +++ b/source/dnode/mnode/impl/test/mnode/mnode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestMnode, 02_Create_Mnode_Invalid_Id) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -63,9 +63,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Invalid_Id) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -98,9 +98,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -115,9 +115,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -132,9 +132,9 @@ TEST_F(MndTestMnode, 04_Create_Mnode) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -148,9 +148,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); @@ -163,9 +163,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -177,9 +177,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -198,9 +198,9 @@ TEST_F(MndTestMnode, 03_Create_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -218,9 +218,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); @@ -233,9 +233,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -247,9 +247,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMDropMnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -268,9 +268,9 @@ TEST_F(MndTestMnode, 04_Drop_Mnode_Rollback) { SMCreateMnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 57b38e55c1..a3bcb7654e 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -61,9 +61,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -77,9 +77,9 @@ TEST_F(MndTestQnode, 02_Create_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,9 +110,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -126,9 +126,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(MndTestQnode, 03_Drop_Qnode) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -158,9 +158,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); taosMsleep(1000); @@ -176,9 +176,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -190,9 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -212,9 +212,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -231,9 +231,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); @@ -246,9 +246,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -259,9 +259,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -280,9 +280,9 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (pRsp->code == 0) break; diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 1828fbd570..ac48eee4d0 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -48,9 +48,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -61,9 +61,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -77,9 +77,9 @@ TEST_F(MndTestSnode, 02_Create_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,9 +110,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -126,9 +126,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -142,9 +142,9 @@ TEST_F(MndTestSnode, 03_Drop_Snode) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -158,9 +158,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); @@ -173,9 +173,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -187,9 +187,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -208,9 +208,9 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -228,9 +228,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); @@ -243,9 +243,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -257,9 +257,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq dropReq = {0}; dropReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &dropReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &dropReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &dropReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -278,9 +278,9 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/trans/trans1.cpp b/source/dnode/mnode/impl/test/trans/trans1.cpp index 5a470fc900..92a442aa5e 100644 --- a/source/dnode/mnode/impl/test/trans/trans1.cpp +++ b/source/dnode/mnode/impl/test/trans/trans1.cpp @@ -112,9 +112,9 @@ TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -129,9 +129,9 @@ TEST_F(MndTestTrans1, 02_Create_Qnode1_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 1; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -165,9 +165,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); @@ -209,9 +209,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -232,9 +232,9 @@ TEST_F(MndTestTrans1, 03_Create_Qnode2_Crash) { SMCreateQnodeReq createReq = {0}; createReq.dnodeId = 2; - int32_t contLen = tSerializeSCreateDropMQSBNodeReq(NULL, 0, &createReq); + int32_t contLen = tSerializeSCreateDropMQSNodeReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); - tSerializeSCreateDropMQSBNodeReq(pReq, contLen, &createReq); + tSerializeSCreateDropMQSNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index be93bd2f59..d4db4709ca 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -132,7 +132,6 @@ typedef enum { SDB_MNODE = 2, SDB_QNODE = 3, SDB_SNODE = 4, - SDB_BNODE = 5, SDB_DNODE = 6, SDB_USER = 7, SDB_AUTH = 8, @@ -403,6 +402,10 @@ const char *sdbStatusName(ESdbStatus status); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); int32_t sdbGetIdFromRaw(SSdb *pSdb, SSdbRaw *pRaw); +void sdbWriteLock(SSdb *pSdb, int32_t type); +void sdbReadLock(SSdb *pSdb, int32_t type); +void sdbUnLock(SSdb *pSdb, int32_t type); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 5393c42da3..e73fd28e71 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -181,3 +181,23 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config pSdb->applyIndex, pSdb->applyTerm, pSdb->applyConfig, *index, *term, *config); #endif } + +void sdbWriteLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d start write lock:%p", type, pLock); + taosThreadRwlockWrlock(pLock); + // mTrace("sdb table:%d stop write lock:%p", type, pLock); +} + +void sdbReadLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d start read lock:%p", type, pLock); + taosThreadRwlockRdlock(pLock); + // mTrace("sdb table:%d stop read lock:%p", type, pLock); +} + +void sdbUnLock(SSdb *pSdb, int32_t type) { + TdThreadRwlock *pLock = &pSdb->locks[type]; + // mTrace("sdb table:%d unlock:%p", type, pLock); + taosThreadRwlockUnlock(pLock); +} diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 18dbea150b..5eedcc545a 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -363,9 +363,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mInfo("write %s to sdb file, total %d rows", sdbTableName(i), sdbGetSize(pSdb, i)); - SHashObj *hash = pSdb->hashObjs[i]; - TdThreadRwlock *pLock = &pSdb->locks[i]; - taosThreadRwlockWrlock(pLock); + SHashObj *hash = pSdb->hashObjs[i]; + sdbWriteLock(pSdb, i); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -410,7 +409,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { sdbFreeRaw(pRaw); ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, i); } if (code == 0) { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 0fee0fbca1..6c00a654e9 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -30,8 +30,6 @@ const char *sdbTableName(ESdbType type) { return "qnode"; case SDB_SNODE: return "snode"; - case SDB_BNODE: - return "bnode"; case SDB_DNODE: return "dnode"; case SDB_USER: @@ -133,12 +131,12 @@ static int32_t sdbGetkeySize(SSdb *pSdb, ESdbType type, const void *pKey) { } static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; @@ -149,7 +147,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * sdbPrintOper(pSdb, pRow, "insert"); if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; @@ -164,12 +162,12 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pRow->pObj, keySize); sdbFreeRow(pSdb, pRow, false); terrno = code; - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return terrno; } } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); @@ -183,26 +181,27 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pNewRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pNewRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pNewRow->type; + sdbWriteLock(pSdb, type); SSdbRow **ppOldRow = taosHashGet(hash, pNewRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return sdbInsertRow(pSdb, hash, pRaw, pNewRow, keySize); } SSdbRow *pOldRow = *ppOldRow; pOldRow->status = pRaw->status; sdbPrintOper(pSdb, pOldRow, "update"); + sdbUnLock(pSdb, type); int32_t code = 0; - SdbUpdateFp updateFp = pSdb->updateFps[pNewRow->type]; + SdbUpdateFp updateFp = pSdb->updateFps[type]; if (updateFp != NULL) { code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - taosThreadRwlockUnlock(pLock); + // sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; @@ -210,12 +209,12 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * } static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *pRow, int32_t keySize) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return terrno; @@ -228,7 +227,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosHashRemove(hash, pOldRow->pObj, keySize); pSdb->tableVer[pOldRow->type]++; - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); sdbFreeRow(pSdb, pRow, false); @@ -282,12 +281,11 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { void *pRet = NULL; int32_t keySize = sdbGetkeySize(pSdb, type, pKey); - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashGet(hash, pKey, keySize); if (ppRow == NULL || *ppRow == NULL) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return NULL; } @@ -310,13 +308,13 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { break; } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return pRet; } static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockWrlock(pLock); + int32_t type = pRow->type; + sdbWriteLock(pSdb, type); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "check"); @@ -324,7 +322,7 @@ static void sdbCheckRow(SSdb *pSdb, SSdbRow *pRow) { sdbFreeRow(pSdb, pRow, true); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { @@ -333,9 +331,9 @@ void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); if (pRow->type >= SDB_MAX) return; - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; + int32_t type = pRow->type; if (lock) { - taosThreadRwlockWrlock(pLock); + sdbWriteLock(pSdb, type); } int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); @@ -345,7 +343,7 @@ void sdbReleaseLock(SSdb *pSdb, void *pObj, bool lock) { } if (lock) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } } @@ -357,8 +355,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, pIter); while (ppRow != NULL) { @@ -373,7 +370,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { *ppObj = pRow->pObj; break; } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return ppRow; } @@ -384,9 +381,8 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return NULL; - TdThreadRwlock *pLock = &pSdb->locks[type]; if (lock) { - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); } SSdbRow **ppRow = taosHashIterate(hash, pIter); @@ -404,7 +400,7 @@ void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStat break; } if (lock) { - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } return ppRow; @@ -416,18 +412,17 @@ void sdbCancelFetch(SSdb *pSdb, void *pIter) { SHashObj *hash = sdbGetHash(pSdb, pRow->type); if (hash == NULL) return; - TdThreadRwlock *pLock = &pSdb->locks[pRow->type]; - taosThreadRwlockRdlock(pLock); + int32_t type = pRow->type; + sdbReadLock(pSdb, type); taosHashCancelIterate(hash, pIter); - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2, void *p3) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -443,17 +438,16 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2 ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); } int32_t sdbGetSize(SSdb *pSdb, ESdbType type) { SHashObj *hash = sdbGetHash(pSdb, type); if (hash == NULL) return 0; - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); int32_t size = taosHashGetSize(hash); - taosThreadRwlockUnlock(pLock); + sdbUnLock(pSdb, type); return size; } @@ -465,9 +459,7 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { if (pSdb->keyTypes[type] != SDB_KEY_INT32) return -1; int32_t maxId = 0; - - TdThreadRwlock *pLock = &pSdb->locks[type]; - taosThreadRwlockRdlock(pLock); + sdbReadLock(pSdb, type); SSdbRow **ppRow = taosHashIterate(hash, NULL); while (ppRow != NULL) { @@ -477,8 +469,7 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) { ppRow = taosHashIterate(hash, ppRow); } - taosThreadRwlockUnlock(pLock); - + sdbUnLock(pSdb, type); maxId = TMAX(maxId, pSdb->maxId[type]); return maxId + 1; } diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index dade85b12d..50e3c3a9c1 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -258,6 +258,7 @@ enum { TD_FTYPE_RSMA_QTASKINFO = 0, }; +#if 0 struct STFile { uint8_t state; STFInfo info; @@ -287,6 +288,7 @@ int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); void tdDestroyTFile(STFile *pTFile); +#endif void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index f96afe6fba..7f418439a8 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -115,7 +115,6 @@ typedef struct { typedef struct { SMqDataRsp dataRsp; - SMqRspHead rspHead; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo pInfo; } STqPushEntry; @@ -150,7 +149,7 @@ int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); // tqRead -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); @@ -182,7 +181,8 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t ver); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 84ce087e55..efd0220a7e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -675,7 +675,7 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv SSchemaWrapper *pSchemaWrapper = &schema; tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, pSchemaWrapper); + (void)tDecodeSSchemaWrapper(&dc, pSchemaWrapper); tDecoderClear(&dc); tdbFree(pData); @@ -1059,7 +1059,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (param->val == NULL) { metaError("vgId:%d, failed to filter NULL data", TD_VID(pMeta->pVnode)); - return -1; + goto END; } else { if (IS_VAR_DATA_TYPE(param->type)) { tagData = varDataVal(param->val); @@ -1111,27 +1111,25 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { break; } } - if (p->suid != pKey->suid) { + if (p == NULL || p->suid != pKey->suid) { break; } first = false; - if (p != NULL) { - int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); - if (cmp == 0) { - // match - tb_uid_t tuid = 0; - if (IS_VAR_DATA_TYPE(pKey->type)) { - tuid = *(tb_uid_t *)(p->data + varDataTLen(p->data)); - } else { - tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); - } - taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // not match but should continue to iter + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (cmp == 0) { + // match + tb_uid_t tuid = 0; + if (IS_VAR_DATA_TYPE(pKey->type)) { + tuid = *(tb_uid_t *)(p->data + varDataTLen(p->data)); } else { - // not match and no more result - break; + tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); } + taosArrayPush(pUids, &tuid); + } else if (cmp == 1) { + // not match but should continue to iter + } else { + // not match and no more result + break; } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 97b0be07c1..2d055acd2f 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -51,7 +51,7 @@ static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema return -1; } - strcpy(pMetaRsp->tbName, tbName); + tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->sversion = pSchema->version; @@ -116,9 +116,10 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const indexMultiTermAdd(terms, term); } } - taosArrayDestroy(pTagVals); indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); + + taosArrayDestroy(pTagVals); #endif return 0; } @@ -159,6 +160,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); type = TSDB_DATA_TYPE_VARCHAR; term = indexTermCreate(suid, DEL_VALUE, type, key, nKey, val, len); + taosMemoryFree(val); } else if (pTagVal->nData == 0) { term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, pTagVal->pData, 0); } @@ -177,6 +179,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche } indexJsonPut(pMeta->pTagIvtIdx, terms, tuid); indexMultiTermDestroy(terms); + taosArrayDestroy(pTagVals); #endif return 0; } @@ -817,6 +820,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA const void *pData = NULL; int nData = 0; + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + // search name index ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); if (ret < 0) { @@ -939,6 +947,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } + ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); @@ -947,7 +956,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tDecoderClear(&dc1); tDecoderClear(&dc2); - if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); + taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); tdbTbcClose(pTbDbc); @@ -1197,8 +1206,8 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { SMetaEntry stbEntry = {0}; STagIdxKey *pTagIdxKey = NULL; int32_t nTagIdxKey; - const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; - const void *pTagData = NULL; // + const SSchema *pTagColumn; + const void *pTagData = NULL; int32_t nTagData = 0; SDecoder dc = {0}; int32_t ret = 0; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index db5f4c55b9..6168a00815 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -17,14 +17,17 @@ extern SSmaMgmt smaMgmt; +#if 0 static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); +#endif static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); +#if 0 /** * @brief Only applicable to Rollup SMA * @@ -48,6 +51,7 @@ int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } * @return int32_t */ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } +#endif /** * @brief Only applicable to Rollup SMA @@ -108,6 +112,7 @@ int32_t smaBegin(SSma *pSma) { return TSDB_CODE_SUCCESS; } +#if 0 /** * @brief pre-commit for rollup sma(sync commit). * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. @@ -169,6 +174,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { #endif return TSDB_CODE_SUCCESS; } +#endif // SQTaskFile ====================================================== @@ -230,6 +236,7 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { return TSDB_CODE_SUCCESS; } +#if 0 /** * @brief post-commit for rollup sma * 1) clean up the outdated qtaskinfo files @@ -249,6 +256,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } +#endif /** * @brief Rsma async commit implementation(only do some necessary light weighted task) diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 722a3f479e..77c5955098 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,8 +15,6 @@ #include "sma.h" -#define RSMA_QTASKINFO_BUFSIZE (32768) // size -#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt #define RSMA_FETCH_DELAY_MAX (120000) // ms @@ -48,23 +46,10 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SR static void tdRSmaFetchTrigger(void *param, void *tmrId); static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo); static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); -static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter); -static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem); static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma); -static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { - // adapt accordingly if definition of SRSmaInfo update - SRSmaInfo *pResult = NULL; - ASSERT(pItem->level == TSDB_RETENTION_L1 || pItem->level == TSDB_RETENTION_L2); - pResult = (SRSmaInfo *)POINTER_SHIFT(pItem, -(sizeof(SRSmaInfoItem) * (pItem->level - 1) + RSMA_INFO_HEAD_LEN)); - ASSERT(pResult->pTSchema->numOfCols > 1); - return pResult; -} - struct SRSmaQTaskInfoItem { int32_t len; int8_t type; @@ -104,12 +89,6 @@ void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, con snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level); } -static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { - return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; -} - -static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } - static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC if (!taskHandle || !(*taskHandle)) return; @@ -803,6 +782,7 @@ static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inpu return TSDB_CODE_SUCCESS; } +#if 0 static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { SSubmitMsgIter msgIter = {0}; SSubmitBlkIter blkIter = {0}; @@ -820,6 +800,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { } return 0; } +#endif /** * @brief sync mode @@ -1189,65 +1170,6 @@ _err: return TSDB_CODE_FAILED; } -static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int8_t type, int64_t qTaskFileVer) { - SVnode *pVnode = pSma->pVnode; - STFile tFile = {0}; - char qTaskInfoFName[TSDB_FILENAME_LEN] = {0}; - - tdRSmaQTaskInfoGetFileName(TD_VID(pVnode), qTaskFileVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - goto _err; - } - - if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) { - if (qTaskFileVer > 0) { - smaWarn("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", not start as %s not exist", - TD_VID(pVnode), type, qTaskFileVer, TD_TFILE_FULL_NAME(&tFile)); - } else { - smaDebug("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", no need as %s not exist", TD_VID(pVnode), - type, qTaskFileVer, TD_TFILE_FULL_NAME(&tFile)); - } - return TSDB_CODE_SUCCESS; - } - - if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) { - goto _err; - } - - STFInfo tFileInfo = {0}; - if (tdLoadTFileHeader(&tFile, &tFileInfo) < 0) { - goto _err; - } - - SRSmaQTaskInfoIter fIter = {0}; - if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) { - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - goto _err; - } - - if (tdRSmaQTaskInfoRestore(pSma, type, &fIter) < 0) { - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - goto _err; - } - - tdRSmaQTaskInfoIterDestroy(&fIter); - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - - // restored successfully from committed or sync - smaInfo("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", qtaskinfo reload succeed", TD_VID(pVnode), - type, qTaskFileVer); - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, restore rsma task %" PRIi8 " for version %" PRIi64 ", qtaskinfo reload failed since %s", - TD_VID(pVnode), type, qTaskFileVer, terrstr()); - return TSDB_CODE_FAILED; -} - /** * @brief reload ts data from checkpoint * @@ -1270,19 +1192,12 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) return TSDB_CODE_SUCCESS; } -#if 0 - // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore - if (tdRSmaRestoreQTaskInfoReload(pSma, type, qtaskFileVer) < 0) { - goto _err; - } -#endif - - // step 3: reload ts data from checkpoint + // step 2: reload ts data from checkpoint if (tdRSmaRestoreTSDataReload(pSma) < 0) { goto _err; } - // step 4: open SRSmaFS for qTaskFiles + // step 3: open SRSmaFS for qTaskFiles if (tdRSmaFSOpen(pSma, qtaskFileVer) < 0) { goto _err; } @@ -1295,191 +1210,6 @@ _err: return TSDB_CODE_FAILED; } -/** - * @brief Restore from SRSmaQTaskInfoItem - * - * @param pSma - * @param pItem - * @return int32_t - */ -static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { - SRSmaInfo *pRSmaInfo = NULL; - void *qTaskInfo = NULL; - - pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pItem->suid); - if (!pRSmaInfo) { - smaDebug("vgId:%d, no restore as no rsma info for table:%" PRIu64, SMA_VID(pSma), pItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (pItem->type == TSDB_RETENTION_L1) { - qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 0); - } else if (pItem->type == TSDB_RETENTION_L2) { - qTaskInfo = RSMA_INFO_QTASK(pRSmaInfo, 1); - } else { - ASSERT(0); - } - - if (!qTaskInfo) { - tdReleaseRSmaInfo(pSma, pRSmaInfo); - smaDebug("vgId:%d, no restore as NULL rsma qTaskInfo for table:%" PRIu64, SMA_VID(pSma), pItem->suid); - return TSDB_CODE_SUCCESS; - } - - if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { - tdReleaseRSmaInfo(pSma, pRSmaInfo); - smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, - pItem->type, terrstr()); - return TSDB_CODE_FAILED; - } - smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, - pItem->type); - - tdReleaseRSmaInfo(pSma, pRSmaInfo); - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile) { - memset(pIter, 0, sizeof(*pIter)); - pIter->pTFile = pTFile; - pIter->offset = TD_FILE_HEAD_SIZE; - - if (tdGetTFileSize(pTFile, &pIter->fsize) < 0) { - return TSDB_CODE_FAILED; - } - - if ((pIter->fsize - TD_FILE_HEAD_SIZE) < RSMA_QTASKINFO_BUFSIZE) { - pIter->nAlloc = pIter->fsize - TD_FILE_HEAD_SIZE; - } else { - pIter->nAlloc = RSMA_QTASKINFO_BUFSIZE; - } - - if (pIter->nAlloc < TD_FILE_HEAD_SIZE) { - pIter->nAlloc = TD_FILE_HEAD_SIZE; - } - - pIter->pBuf = taosMemoryMalloc(pIter->nAlloc); - if (!pIter->pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - pIter->qBuf = pIter->pBuf; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish) { - STFile *pTFile = pIter->pTFile; - int64_t nBytes = RSMA_QTASKINFO_BUFSIZE; - - if (pIter->offset >= pIter->fsize) { - *isFinish = true; - return TSDB_CODE_SUCCESS; - } - - if ((pIter->fsize - pIter->offset) < RSMA_QTASKINFO_BUFSIZE) { - nBytes = pIter->fsize - pIter->offset; - } - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { - return TSDB_CODE_FAILED; - } - - int32_t infoLen = 0; - taosDecodeFixedI32(pIter->pBuf, &infoLen); - if (infoLen > nBytes) { - if (infoLen <= RSMA_QTASKINFO_BUFSIZE) { - terrno = TSDB_CODE_RSMA_FILE_CORRUPTED; - smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); - return TSDB_CODE_FAILED; - } - if (pIter->nAlloc < infoLen) { - pIter->nAlloc = infoLen; - void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); - if (!pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - pIter->pBuf = pBuf; - } - - nBytes = infoLen; - - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { - return TSDB_CODE_FAILED; - } - - if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { - return TSDB_CODE_FAILED; - } - } - - pIter->qBuf = pIter->pBuf; - pIter->offset += nBytes; - pIter->nBytes = nBytes; - pIter->nBufPos = 0; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter) { - while (1) { - // block iter - bool isFinish = false; - if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) { - return TSDB_CODE_FAILED; - } - if (isFinish) { - return TSDB_CODE_SUCCESS; - } - - // consume the block - int32_t qTaskInfoLenWithHead = 0; - pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); - if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - smaError("vgId:%d, restore rsma task %" PRIi8 " from qtaskinfo file %s failed since %s", SMA_VID(pSma), type, - TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); - return TSDB_CODE_FAILED; - } - - while (1) { - if ((pIter->nBufPos + qTaskInfoLenWithHead) <= pIter->nBytes) { - SRSmaQTaskInfoItem infoItem = {0}; - pIter->qBuf = taosDecodeFixedI8(pIter->qBuf, &infoItem.type); - pIter->qBuf = taosDecodeFixedI64(pIter->qBuf, &infoItem.suid); - infoItem.qTaskInfo = pIter->qBuf; - infoItem.len = tdRSmaQTaskInfoContLen(qTaskInfoLenWithHead); - // do the restore job - smaDebug("vgId:%d, restore rsma task %" PRIi8 " from qtaskinfo file %s offset:%" PRIi64 "\n", SMA_VID(pSma), - type, TD_TFILE_FULL_NAME(pIter->pTFile), pIter->offset - pIter->nBytes + pIter->nBufPos); - tdRSmaQTaskInfoItemRestore(pSma, &infoItem); - - pIter->qBuf = POINTER_SHIFT(pIter->qBuf, infoItem.len); - pIter->nBufPos += qTaskInfoLenWithHead; - - if ((pIter->nBufPos + RSMA_QTASKINFO_HEAD_LEN) >= pIter->nBytes) { - // prepare and load next block in the file - pIter->offset -= (pIter->nBytes - pIter->nBufPos); - break; - } - - pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead); - continue; - } - // prepare and load next block in the file - pIter->offset -= (pIter->nBytes - pIter->nBufPos); - break; - } - } - - return TSDB_CODE_SUCCESS; -} - int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; @@ -1523,148 +1253,6 @@ _err: return TSDB_CODE_FAILED; } -#if 0 -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - int32_t vid = SMA_VID(pSma); - int64_t toffset = 0; - bool isFileCreated = false; - - if (taosHashGetSize(pInfoHash) <= 0) { - return TSDB_CODE_SUCCESS; - } - - void *infoHash = taosHashIterate(pInfoHash, NULL); - if (!infoHash) { - return TSDB_CODE_SUCCESS; - } - - int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); - if (pRSmaStat->commitAppliedVer <= fsMaxVer) { - smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, - pRSmaStat->commitAppliedVer, fsMaxVer); - return TSDB_CODE_SUCCESS; - } - - STFile tFile = {0}; -#if 0 - if (pRSmaStat->commitAppliedVer > 0) { - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); - goto _err; - } - if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); - goto _err; - } - smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile)); - - isFileCreated = true; - } -#endif - - while (infoHash) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - infoHash = taosHashIterate(pInfoHash, infoHash); - continue; - } - - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { -#if 0 - qTaskInfo_t taskInfo = RSMA_INFO_IQTASK(pRSmaInfo, i); -#endif - qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pRSmaInfo, i); - if (!taskInfo) { - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); - continue; - } - - char *pOutput = NULL; - int32_t len = 0; - int8_t type = (int8_t)(i + 1); - if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { - smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr()); - goto _err; - } - if (!pOutput || len <= 0) { - smaDebug("vgId:%d, rsma, table %" PRIi64 - " level %d serialize qTaskInfo success but no output(len %d), not persist", - vid, pRSmaInfo->suid, i + 1, len); - taosMemoryFreeClear(pOutput); - continue; - } - - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo success with len %d, need persist", vid, - pRSmaInfo->suid, i + 1, len); - - if (!isFileCreated) { - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); - if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); - goto _err; - } - if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); - goto _err; - } - smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid, - TD_TFILE_FULL_NAME(&tFile)); - - isFileCreated = true; - } - - char tmpBuf[RSMA_QTASKINFO_HEAD_LEN] = {0}; - void *pTmpBuf = &tmpBuf; - int32_t headLen = 0; - headLen += taosEncodeFixedI32(&pTmpBuf, len + RSMA_QTASKINFO_HEAD_LEN); - headLen += taosEncodeFixedI8(&pTmpBuf, type); - headLen += taosEncodeFixedI64(&pTmpBuf, pRSmaInfo->suid); - - ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN); - tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset); - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid, - pRSmaInfo->suid, i + 1, headLen, toffset); - tdAppendTFile(&tFile, pOutput, len, &toffset); - smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, - pRSmaInfo->suid, i + 1, len, toffset); - - taosMemoryFree(pOutput); - } - - infoHash = taosHashIterate(pInfoHash, infoHash); - } - - if (isFileCreated) { - if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), - tstrerror(terrno)); - goto _err; - } else { - smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); - } - - tdCloseTFile(&tFile); - tdDestroyTFile(&tFile); - } - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); - if (isFileCreated) { - tdRemoveTFile(&tFile); - tdDestroyTFile(&tFile); - } - return TSDB_CODE_FAILED; -} - -#endif - /** * @brief trigger to get rsma result in async mode * diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 6d7b7df1ee..4d09d690d6 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -16,7 +16,7 @@ #include "sma.h" // smaFileUtil ================ - +#if 0 #define TD_FILE_STATE_OK 0 #define TD_FILE_STATE_BAD 1 @@ -182,6 +182,8 @@ void tdCloseTFile(STFile *pTFile) { void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); } +#endif + void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName) { if (version < 0) { @@ -221,6 +223,7 @@ void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool e } } +#if 0 int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) { TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); TD_TFILE_SET_CLOSED(pTFile); @@ -286,6 +289,8 @@ int32_t tdRemoveTFile(STFile *pTFile) { return 0; } +#endif + // smaXXXUtil ================ void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e1578b37f1..6d71496008 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -192,7 +192,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { return -1; } - memcpy(buf, &pPushEntry->rspHead, sizeof(SMqRspHead)); + memcpy(buf, &pPushEntry->dataRsp.head, sizeof(SMqRspHead)); void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); @@ -215,7 +215,7 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:%" PRId64 ", (epoch %d) push rsp, block num: %d, reqOffset:%s, rspOffset:%s", - TD_VID(pTq->pVnode), pPushEntry->rspHead.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -560,9 +560,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { memcpy(pPushEntry->subKey, pHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); dataRsp.withTbName = 0; memcpy(&pPushEntry->dataRsp, &dataRsp, sizeof(SMqDataRsp)); - pPushEntry->rspHead.consumerId = consumerId; - pPushEntry->rspHead.epoch = reqEpoch; - pPushEntry->rspHead.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + pPushEntry->dataRsp.head.consumerId = consumerId; + pPushEntry->dataRsp.head.epoch = reqEpoch; + pPushEntry->dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey) + 1, &pPushEntry, sizeof(void*)); tqDebug("tmq poll: consumer %ld, subkey %s, vg %d save handle to push mgr", consumerId, pHandle->subKey, TD_VID(pTq->pVnode)); @@ -595,7 +595,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitTaosxRsp(&taosxRsp, pReq); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - tqScan(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); + tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (metaRsp.metaRspLen > 0) { if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { @@ -924,7 +924,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqTableSink; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); @@ -1010,16 +1010,21 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { qError("stream task input del failed, task id %d", pTask->taskId); + + taosFreeQitem(pRefBlock); continue; } + if (streamSchedExec(pTask) < 0) { qError("stream task launch failed, task id %d", pTask->taskId); continue; } + } else { streamTaskInputFail(pTask); } } + int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 58d051bec1..305ee82982 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -123,7 +123,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs return 0; } -int32_t tqScan(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 521d12fdab..79d7e27642 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -284,7 +284,251 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } -void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { +void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper; + + int32_t blockSz = taosArrayGetSize(pBlocks); + + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; + } + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + for (int32_t i = 0; i < blockSz; i++) { + bool createTb = true; + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(serializedDeleteReq); + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + char* ctbName = NULL; + // set child table name + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + } + + int32_t schemaLen = 0; + void* schemaStr = NULL; + + int64_t uid = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq createTbReq = {0}; + + // set const + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + createTbReq.name = ctbName; + ctbName = NULL; + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + taosArrayPush(tagArray, &tagVal); + createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(tagArray); + tdDestroySVCreateTbReq(&createTbReq); + return; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + createTbReq.ctb.tagName = tagName; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + // set schema str + schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + tEncoderClear(&encoder); + taosMemoryFree(schemaStr); + return; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %ld, actual suid %ld", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + createTb = false; + uid = mr.me.uid; + metaReaderClear(&mr); + + tqDebug("vgId:%d, stream write, table %s, uid %ld already exist, skip create", TD_VID(pVnode), ctbName, uid); + + taosMemoryFreeClear(ctbName); + } + + int32_t cap = sizeof(SSubmitReq); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + + SSubmitReq* ret = rpcMallocCont(cap); + ret->header.vgId = pVnode->config.vgId; + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(1); + + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + + blkHead->numOfRows = htonl(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + blkHead->schemaLen = 0; + + tqDebug("tq sink, convert block %d, rows: %d", i, rows); + + int32_t dataLen = 0; + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); + STSRow* rowData = blkSchema; + if (createTb) { + memcpy(blkSchema, schemaStr, schemaLen); + blkHead->schemaLen = htonl(schemaLen); + rowData = POINTER_SHIFT(blkSchema, schemaLen); + } else { + blkHead->uid = htobe64(uid); + } + + taosMemoryFreeClear(schemaStr); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); + } else { + void* colData = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); + } + } + tdSRowEnd(&rb); + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + dataLen += rowLen; + } + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + ret->length = htonl(ret->length); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = ret, + .contLen = ntohl(ret->length), + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + rpcFreeCont(ret); + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } + taosArrayDestroy(tagArray); +} + +void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; SBatchDeleteReq deleteReq = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index aca9042261..a4d993bc6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1045,7 +1045,9 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - ASSERT(eno == 0); + ASSERT(eno == 0 && + "tsdbCommit failure" + "Restart taosd"); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index d08b743763..6fd5629592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -819,7 +819,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { nRef = atomic_sub_fetch_32(&fSet.pHeadF->nRef, 1); if (nRef == 0) { tsdbHeadFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pHeadF, fname); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); taosMemoryFree(fSet.pHeadF); } } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 450032d4f2..0189ced3c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -160,6 +160,7 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid goto _err; } + ASSERT(pPool != NULL); // do delete SDelData *pDelData = (SDelData *)vnodeBufPoolMalloc(pPool, sizeof(*pDelData)); if (pDelData == NULL) { @@ -353,6 +354,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel; + ASSERT(pPool != NULL); pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2); if (pTbData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -492,6 +494,7 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN // node level = tsdbMemSkipListRandLevel(&pTbData->sl); + ASSERT(pPool != NULL); pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow)); if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 09a46b59bb..2f4fdfc5f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -133,8 +133,15 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { id.uid = pIter->uid; } - tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + code = tBlockDataInit(pBlock, &id, pInfo->pSchema, pInfo->colIds, pInfo->numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } double el = (taosGetTimestampUs() - st) / 1000.0; pInfo->elapsedTime += el; @@ -144,9 +151,6 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { ", last file index:%d, last block index:%d, entry:%d, %p, elapsed time:%.2f ms, %s", pInfo->loadBlocks, pIter->uid, pIter->iStt, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock, el, idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _exit; - } pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; tsdbDebug("last block index list:%d, %d, %s", pInfo->blockIndex[0], pInfo->blockIndex[1], idStr); @@ -459,8 +463,8 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { - int32_t code = 0; int32_t step = pIter->backward ? -1 : 1; + terrno = TSDB_CODE_SUCCESS; // no qualified last file block in current file, no need to fetch row if (pIter->pSttBlk == NULL) { @@ -469,6 +473,10 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { int32_t iBlockL = pIter->iSttBlk; SBlockData *pBlockData = loadLastBlock(pIter, idStr); + if (pBlockData == NULL && terrno != TSDB_CODE_SUCCESS) { + goto _exit; + } + pIter->iRow += step; while (1) { @@ -494,11 +502,7 @@ bool tLDataIterNextRow(SLDataIter *pIter, const char *idStr) { pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); _exit: - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - } - - return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); + return (terrno == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); } SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 6b706901ba..3f88d478e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -340,7 +340,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb pIter->pLastBlockReader = taosMemoryCalloc(1, sizeof(struct SLastBlockReader)); if (pIter->pLastBlockReader == NULL) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; - tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); + tsdbError("failed to prepare the last block iterator, code:%s %s", tstrerror(code), pReader->idStr); return code; } } @@ -645,11 +645,11 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " - "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, - pReader->idStr); + tsdbDebug("load block of %" PRIzu + " tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " + "time:%.2f ms %s", + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, + pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1506,7 +1506,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* if (pReader->pMemSchema == NULL) { int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - return pReader->pMemSchema; + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + return pReader->pMemSchema; + } } if (pReader->pMemSchema->version == sversion) { @@ -1515,7 +1520,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* taosMemoryFree(pReader->pMemSchema); int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &pReader->pMemSchema); - return pReader->pMemSchema; + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } else { + return pReader->pMemSchema; + } } static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, @@ -1569,7 +1579,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1579,7 +1592,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1590,16 +1606,29 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } } else { if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == tsLast) { @@ -1608,7 +1637,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1618,7 +1650,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow); } else { init = true; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1652,13 +1687,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { return TSDB_CODE_SUCCESS; } else { - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1669,7 +1707,11 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, tRowMergerClear(&merge); } } else { // not merge block data - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); ASSERT(mergeBlockData); @@ -1678,7 +1720,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1714,7 +1756,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader STSRow* pTSRow = NULL; SRowMerger merge = {0}; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); @@ -1722,7 +1768,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1746,9 +1792,9 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; - STSRow* pTSRow = NULL; - + SRowMerger merge = {0}; + STSRow* pTSRow = NULL; + int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; @@ -1811,7 +1857,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -1821,8 +1871,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1832,9 +1886,17 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, + &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } if (minKey == k.ts) { @@ -1842,16 +1904,31 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); + code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } } else { if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } if (minKey == ik.ts) { @@ -1860,9 +1937,16 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); + code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; } - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); } if (minKey == tsLast) { @@ -1871,7 +1955,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tRowMerge(&merge, &fRow1); } else { init = true; - tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge); } @@ -1879,7 +1966,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } else { tRowMerge(&merge, &fRow); } @@ -1887,7 +1977,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2048,9 +2138,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc STSRow* pTSRow = NULL; SRowMerger merge = {0}; - tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + code = tRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2676,9 +2770,9 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); - int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1 - : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000 - : 1000000); + int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI) ? 1L + : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L + : 1000000L); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -2888,6 +2982,10 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe } STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } + tRowMergerAdd(pMerger, pRow, pTSchema); } @@ -3041,19 +3139,34 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SRowMerger merge = {0}; // get the correct schema for data in memory + terrno = 0; STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } if (pReader->pSchema == NULL) { pReader->pSchema = pTSchema; } - tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } + tRowMergerAdd(&merge, pNextRow, pTSchema1); - doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); - int32_t code = tRowMergerGetRow(&merge, pTSRow); + code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tRowMergerGetRow(&merge, pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3073,19 +3186,44 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, piRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, piRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } tRowMerge(&merge, pRow); - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = tRowMergerInit(&merge, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, + pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } int32_t code = tRowMergerGetRow(&merge, pTSRow); @@ -3372,14 +3510,13 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:%" PRId64 " , %s", pReader->suid, -1, - pReader->idStr); + tsdbError("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->suid, pReader->idStr); } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1); if (pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:%" PRId64 " , %s", pKey->uid, -1, pReader->idStr); + tsdbError("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr); } } @@ -3410,18 +3547,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl STsdbReader* pPrevReader = pReader->innerReader[0]; STsdbReader* pNextReader = pReader->innerReader[1]; - // we need only one row - pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; - pPrevReader->pReadSnap = pReader->pReadSnap; + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->pSchema = pReader->pSchema; + pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pReadSnap = pReader->pReadSnap; - pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; - pNextReader->pReadSnap = pReader->pReadSnap; + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->pSchema = pReader->pSchema; + pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pReadSnap = pReader->pReadSnap; code = doOpenReaderImpl(pPrevReader); if (code != TSDB_CODE_SUCCESS) { @@ -3444,7 +3581,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl return code; _err: - tsdbError("failed to create data reader, code:%s %s", tstrerror(code), pReader->idStr); + tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); return code; } @@ -3733,7 +3870,7 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { - if (isEmptyQueryTimeWindow(&pReader->window)) { + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 47c836d9c1..4e02a28cdf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -719,8 +719,6 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { } pMerger->version = key.version; - -_exit: return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 730bd264a7..682f2b4225 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -112,6 +112,8 @@ void vnodeBufPoolReset(SVBufPool *pPool) { void *vnodeBufPoolMalloc(SVBufPool *pPool, int size) { SVBufPoolNode *pNode; void *p = NULL; + ASSERT(pPool != NULL); + taosThreadSpinLock(&pPool->lock); if (pPool->node.size >= pPool->ptr - pPool->node.data + size) { // allocate from the anchor node diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 07d9b96261..07c4c32955 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -73,7 +73,7 @@ int vnodeBegin(SVnode *pVnode) { int vnodeShouldCommit(SVnode *pVnode) { if (pVnode->inUse) { - return pVnode->inUse->size > pVnode->inUse->node.size; + return osDataSpaceAvailable() && (pVnode->inUse->size > pVnode->inUse->node.size); } return false; } @@ -89,6 +89,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { data = NULL; if (vnodeEncodeInfo(pInfo, &data) < 0) { + vError("failed to encode json info."); return -1; } @@ -101,7 +102,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { } if (taosWriteFile(pFile, data, strlen(data)) < 0) { - vError("failed to write info file:%s data:%s", fname, terrstr()); + vError("failed to write info file:%s error:%s", fname, terrstr()); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -233,15 +234,15 @@ int vnodeCommit(SVnode *pVnode) { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } if (vnodeSaveInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to save vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit // smaSyncPreCommit(pVnode->pSma); - if (smaAsyncPreCommit(pVnode->pSma) < 0) { - ASSERT(0); + if(smaAsyncPreCommit(pVnode->pSma) < 0){ + vError("vgId:%d, failed to async pre-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -250,44 +251,44 @@ int vnodeCommit(SVnode *pVnode) { // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (VND_IS_RSMA(pVnode)) { if (smaAsyncCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma0 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA1(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } if (tsdbCommit(VND_RSMA2(pVnode)) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } else { if (tsdbCommit(pVnode->pTsdb) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } } if (tqCommit(pVnode->pTq) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit tq since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } // walCommit (TODO) // commit info if (vnodeCommitInfo(dir, &info) < 0) { - ASSERT(0); + vError("vgId:%d, failed to commit vnode info since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -296,7 +297,7 @@ int vnodeCommit(SVnode *pVnode) { // postCommit // smaSyncPostCommit(pVnode->pSma); if (smaAsyncPostCommit(pVnode->pSma) < 0) { - ASSERT(0); + vError("vgId:%d, failed to async post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 298653d3ed..001bb5f7f2 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -140,7 +140,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg)); if (pVnode->pWal == NULL) { - vError("vgId:%d, failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno)); + vError("vgId:%d, failed to open vnode wal since %s. wal:%s", TD_VID(pVnode), tstrerror(terrno), tdir); goto _err; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index e18379fe8a..64df3aa1eb 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -272,6 +272,12 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { SRpcMsg rspMsg = {0}; void *pRsp = NULL; + if (msgNum >= MAX_META_MSG_IN_BATCH) { + code = TSDB_CODE_INVALID_MSG; + qError("too many msgs %d in vnode batch meta req", msgNum); + goto _exit; + } + SArray *batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); if (NULL == batchRsp) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -324,6 +330,11 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { rspSize += sizeof(int32_t); offset = 0; + if (rspSize > MAX_META_BATCH_RSP_SIZE) { + code = TSDB_CODE_INVALID_MSG_LEN; + goto _exit; + } + pRsp = rpcMallocCont(rspSize); if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b531cf7574..d0a00a8b6d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -41,7 +41,10 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nReqs; tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - tStartDecode(&dc); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } if (tDecodeI32v(&dc, &nReqs) < 0) { code = TSDB_CODE_INVALID_MSG; @@ -166,6 +169,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; + if (!pVnode->inUse) { + terrno = TSDB_CODE_VND_NOT_SYNCED; + vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); + return -1; + } + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); @@ -286,10 +295,16 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change - vnodeCommit(pVnode); + if (vnodeCommit(pVnode) < 0) { + vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one - vnodeBegin(pVnode); + if (vnodeBegin(pVnode) < 0) { + vError("vgId:%d, failed to begin vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } } return 0; @@ -787,15 +802,11 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, tInitSubmitBlkIter(msgIter, pBlock, &blkIter); if (blkIter.row == NULL) return 0; - if (!pSchema || (suid != msgIter->suid) || rv != TD_ROW_SVER(blkIter.row)) { - if (pSchema) { - taosMemoryFreeClear(pSchema); - } - pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema - if (pSchema) { - suid = msgIter->suid; - rv = TD_ROW_SVER(blkIter.row); - } + + pSchema = metaGetTbTSchema(pMeta, msgIter->suid, TD_ROW_SVER(blkIter.row), 1); // TODO: use the real schema + if (pSchema) { + suid = msgIter->suid; + rv = TD_ROW_SVER(blkIter.row); } if (!pSchema) { printf("%s:%d no valid schema\n", tags, __LINE__); diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h index 8edd3b3880..746344b7ed 100644 --- a/source/libs/catalog/inc/ctgRemote.h +++ b/source/libs/catalog/inc/ctgRemote.h @@ -20,6 +20,8 @@ extern "C" { #endif +#define CTG_MAX_REQ_IN_BATCH 1048576 + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b027dbcfa5..218a86ed5c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -173,15 +173,19 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgCloneMetaOutput(output, pOutput)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncReq); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + CTG_RET(code); } @@ -290,15 +294,19 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, rspMsg->tableType == TSDB_SUPER_TABLE, &output->tbMeta)); } - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncOp)); + code = ctgUpdateTbMetaEnqueue(pCtg, output, syncOp); + output = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; _return: - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); - + if (output) { + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + } + CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 1305540a50..93d36bc4b3 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -71,7 +71,7 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; } @@ -202,7 +202,7 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", pJob->queryId, taskIdx, - ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); + ctgTaskTypeStr(task.type), (int32_t)taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; } @@ -252,7 +252,7 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgIndexCtx* ctx = task.taskCtx; - strcpy(ctx->indexFName, name); + tstrncpy(ctx->indexFName, name, sizeof(ctx->indexFName)); taosArrayPush(pJob->pTasks, &task); @@ -277,7 +277,7 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgUdfCtx* ctx = task.taskCtx; - strcpy(ctx->udfName, name); + tstrncpy(ctx->udfName, name, sizeof(ctx->udfName)); taosArrayPush(pJob->pTasks, &task); @@ -1056,7 +1056,6 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1223,7 +1222,6 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; @@ -1309,7 +1307,6 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1346,7 +1343,6 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } _return: @@ -1382,7 +1378,6 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu default: ctgError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); - break; } if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1672,7 +1667,7 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); - ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, (int32_t)taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } @@ -2010,10 +2005,6 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { _return: - if (dbCache) { - ctgReleaseVgInfoToCache(pCtg, dbCache); - } - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3a774afdcc..1a7a0057ba 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -597,6 +597,8 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); + taosMemoryFree(operation->data); + taosMemoryFree(operation); CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -648,6 +650,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -657,7 +660,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->dbId = dbId; op->data = msg; @@ -668,7 +671,6 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -681,6 +683,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -690,7 +693,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); op->data = msg; @@ -700,7 +703,6 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -714,12 +716,13 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->stbName, stbName, sizeof(msg->stbName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->stbName, stbName, sizeof(msg->stbName)); msg->dbId = dbId; msg->suid = suid; @@ -731,7 +734,6 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -744,12 +746,13 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg->pCtg = pCtg; - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); - strncpy(msg->tbName, tbName, sizeof(msg->tbName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->tbName, tbName, sizeof(msg->tbName)); msg->dbId = dbId; op->data = msg; @@ -760,7 +763,6 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, _return: - taosMemoryFreeClear(op->data); CTG_RET(code); } @@ -773,6 +775,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); + taosMemoryFree(op); ctgFreeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -782,7 +785,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } - strncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->pCtg = pCtg; msg->dbId = dbId; msg->dbInfo = dbInfo; @@ -796,8 +799,6 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: ctgFreeVgInfo(dbInfo); - taosMemoryFreeClear(op->data); - taosMemoryFreeClear(op); CTG_RET(code); } @@ -810,12 +811,14 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } char *p = strchr(output->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { - memmove(output->dbFName, p + 1, strlen(p + 1)); + int32_t len = strlen(p + 1); + memmove(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -834,8 +837,6 @@ _return: taosMemoryFree(output); } - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -847,11 +848,12 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateEpsetMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg->pCtg = pCtg; - strcpy(msg->dbFName, dbFName); + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->vgId = vgId; msg->epSet = *pEpSet; @@ -863,8 +865,6 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -877,6 +877,7 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -892,7 +893,6 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp _return: tFreeSGetUserAuthRsp(pAuth); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -906,6 +906,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncO SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -923,7 +924,6 @@ _return: taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); - taosMemoryFreeClear(msg); CTG_RET(code); } @@ -937,6 +937,7 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -952,8 +953,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -968,6 +967,7 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg)); + taosMemoryFree(op); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -981,8 +981,6 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool _return: - taosMemoryFreeClear(msg); - CTG_RET(code); } @@ -1218,7 +1216,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { CTG_CACHE_STAT_INC(numOfDb, 1); SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); @@ -1334,8 +1332,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin metaRent.smaVer = pCache->pIndex->version; } - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); + tstrncpy(metaRent.dbFName, dbFName, sizeof(metaRent.dbFName)); + tstrncpy(metaRent.stbName, tbName, sizeof(metaRent.stbName)); CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); @@ -1393,8 +1391,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); } else { + CTG_LOCK(CTG_WRITE, &pCache->metaLock); taosMemoryFree(pCache->pMeta); pCache->pMeta = meta; + CTG_UNLOCK(CTG_WRITE, &pCache->metaLock); } if (NULL == orig) { @@ -1419,8 +1419,10 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); - + if (pCache) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); + } + return TSDB_CODE_SUCCESS; } @@ -1457,6 +1459,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa return TSDB_CODE_SUCCESS; } + CTG_LOCK(CTG_WRITE, &pCache->indexLock); + if (pCache->pIndex) { if (0 == suid) { suid = pCache->pIndex->suid; @@ -1466,6 +1470,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa } pCache->pIndex = pIndex; + CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); + *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, @@ -1483,7 +1489,9 @@ int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool sync int32_t code = 0; CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq)); + code = ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq); + pOutput = NULL; + CTG_ERR_JRET(code); return TSDB_CODE_SUCCESS; @@ -1585,7 +1593,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { dbCache = NULL; - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); @@ -1675,9 +1683,9 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); - CTG_ERR_JRET( - ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + code = ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize); pMeta->tbMeta = NULL; + CTG_ERR_JRET(code); } if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { @@ -1692,10 +1700,8 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { _return: - if (pMeta) { - taosMemoryFreeClear(pMeta->tbMeta); - taosMemoryFreeClear(pMeta); - } + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); taosMemoryFreeClear(msg); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 744d6eab06..23bccb0835 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -19,6 +19,7 @@ #include "tname.h" #include "tref.h" #include "trpc.h" +#include "ctgRemote.h" int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; @@ -579,6 +580,11 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { int32_t num = taosArrayGetSize(pBatch->pMsgs); SBatchReq* pBatchReq = (SBatchReq*)(*msg); + if (num >= CTG_MAX_REQ_IN_BATCH) { + qError("too many msgs %d in one batch request", num); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + pBatchReq->header.vgId = htonl(vgId); pBatchReq->msgNum = htonl(num); offset += sizeof(SBatchReq); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index ad677a81ff..ce4c4cf09b 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -89,8 +89,8 @@ extern "C" { #define EXPLAIN_STRING_TYPE_FORMAT "%s" #define EXPLAIN_INPUT_ORDER_FORMAT "input_order=%s" #define EXPLAIN_OUTPUT_ORDER_TYPE_FORMAT "output_order=%s" -#define EXPLAIN_OFFSET_FORMAT "offset=%d" -#define EXPLAIN_SOFFSET_FORMAT "soffset=%d" +#define EXPLAIN_OFFSET_FORMAT "offset=%" PRId64 +#define EXPLAIN_SOFFSET_FORMAT "soffset=%" PRId64 #define EXPLAIN_PARTITIONS_FORMAT "partitions=%d" #define COMMAND_RESET_LOG "resetLog" diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 18d839e109..095d2b093d 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -43,7 +43,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe blockEncode(pBlock, (*pRsp)->data, &len, numOfCols, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); - blockDataDestroy(pBlock); return TSDB_CODE_SUCCESS; } @@ -59,21 +58,33 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } } -static SSDataBlock* buildDescResultDataBlock() { +static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); - blockDataAppendColInfo(pBlock, &infoData); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + code = blockDataAppendColInfo(pBlock, &infoData); + } + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + code = blockDataAppendColInfo(pBlock, &infoData); + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - - infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); - blockDataAppendColInfo(pBlock, &infoData); - - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); - blockDataAppendColInfo(pBlock, &infoData); - return pBlock; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } static void setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock, int32_t numOfRows, STableMeta* pMeta) { @@ -109,22 +120,39 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** SDescribeStmt* pDesc = (SDescribeStmt*)pStmt; int32_t numOfRows = TABLE_TOTAL_COL_NUM(pDesc->pMeta); - SSDataBlock* pBlock = buildDescResultDataBlock(); - setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); - - return buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); + SSDataBlock* pBlock = NULL; + int32_t code = buildDescResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + setDescResultIntoDataBlock(sysInfoUser, pBlock, numOfRows, pDesc->pMeta); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, DESCRIBE_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t execResetQueryCache() { return catalogClearCache(); } -static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = createDataBlock(); - SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); - blockDataAppendColInfo(pBlock, &infoData); +static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { + SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - return pBlock; + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { @@ -259,21 +287,37 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S } static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildCreateDBResultDataBlock(); - setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); - return buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); + SSDataBlock* pBlock = NULL; + int32_t code = buildCreateDBResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_DB_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } -static SSDataBlock* buildCreateTbResultDataBlock() { +static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); - blockDataAppendColInfo(pBlock, &infoData); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + if (TSDB_CODE_SUCCESS == code) { + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + code = blockDataAppendColInfo(pBlock, &infoData); + } - infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); - blockDataAppendColInfo(pBlock, &infoData); - - return pBlock; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; } void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { @@ -317,7 +361,12 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { SArray* pTagVals = NULL; STag* pTag = (STag*)pCfg->pTags; - if (pCfg->pTags && tTagIsJson(pTag)) { + if (NULL == pCfg->pTags || pCfg->numOfTags <= 0) { + qError("tag missed in table cfg, pointer:%p, numOfTags:%d", pCfg->pTags, pCfg->numOfTags); + return TSDB_CODE_APP_ERROR; + } + + if (tTagIsJson(pTag)) { char* pJson = parseTagDatatoJson(pTag); if (pJson) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); @@ -482,12 +531,16 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p } static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildCreateTbResultDataBlock(); - int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); - if (code) { - return code; + SSDataBlock* pBlock = NULL; + int32_t code = buildCreateTbResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setCreateTBResultIntoDataBlock(pBlock, pStmt->pDbCfg, pStmt->tableName, pStmt->pTableCfg); } - return buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_TB_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { @@ -556,8 +609,12 @@ _return: return TSDB_CODE_SUCCESS; } -static SSDataBlock* buildLocalVariablesResultDataBlock() { +static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pBlock->info.hasVarCol = true; pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); @@ -572,7 +629,8 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() { infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; taosArrayPush(pBlock->pDataBlock, &infoData); - return pBlock; + *pOutput = pBlock; + return TSDB_CODE_SUCCESS; } int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { @@ -604,12 +662,16 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { } static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { - SSDataBlock* pBlock = buildLocalVariablesResultDataBlock(); - int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); - if (code) { - return code; + SSDataBlock* pBlock = NULL; + int32_t code = buildLocalVariablesResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setLocalVariablesResultIntoDataBlock(pBlock); } - return buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp); + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_LOCAL_VARIABLES_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; } static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) { @@ -659,6 +721,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p if (TSDB_CODE_SUCCESS == code) { code = buildRetrieveTableRsp(pBlock, LIST_LENGTH(pSelect->pProjectionList), pRsp); } + blockDataDestroy(pBlock); return code; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index c5de44e1f6..e0a4218243 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1598,6 +1598,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); + blockDataDestroy(pBlock); QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index fc996a6003..47450328be 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -53,6 +53,11 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) +#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) +#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) +#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) +#define SET_SESSION_WIN_KEY_INVALID(pWinKey) ((pWinKey)->win.skey = INT64_MIN) + enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, @@ -434,15 +439,15 @@ typedef struct SCatchSupporter { } SCatchSupporter; typedef struct SStreamAggSupporter { - SHashObj* pResultRows; - SArray* pCurWins; - int32_t valueSize; - int32_t keySize; - char* pKeyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // buffer page that is active - SSDataBlock* pScanBlock; + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; } SStreamAggSupporter; typedef struct SWindowSupporter { @@ -736,42 +741,54 @@ typedef struct SSessionAggOperatorInfo { } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { - SResultRowPosition pos; - STimeWindow win; - uint64_t groupId; + void* pOutputBuf; + SSessionKey sessionWin; bool isOutput; - bool isClosed; } SResultWindowInfo; typedef struct SStateWindowInfo { SResultWindowInfo winInfo; - SStateKeys stateKey; + SStateKeys* pStateKey; } SStateWindowInfo; typedef struct SStreamSessionAggOperatorInfo { SOptrBasicInfo binfo; SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; - int64_t gap; // session window gap int32_t primaryTsIndex; // primary timestamp slot id int32_t endTsIndex; // window end timestamp slot id int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; // delete result - SSDataBlock* pUpdateRes; // update window + SSDataBlock* pWinBlock; // window result + SSDataBlock* pDelRes; // delete result + SSDataBlock* pUpdateRes; // update window bool returnUpdate; - SHashObj* pStDeleted; + SSHashObj* pStDeleted; void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; SHashObj* pGroupIdTbNameMap; } SStreamSessionAggOperatorInfo; +typedef struct SStreamStateAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + STimeWindowAggSupp twAggSup; + SColumn stateCol; + SSDataBlock* pDelRes; + SSHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; + SHashObj* pGroupIdTbNameMap; +} SStreamStateAggOperatorInfo; + typedef struct SStreamPartitionOperatorInfo { SOptrBasicInfo binfo; SPartitionBySupporter partitionSup; @@ -834,24 +851,6 @@ typedef struct SStateWindowOperatorInfo { const SNode* pCondition; } SStateWindowOperatorInfo; -typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SColumn stateCol; - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; - SHashObj* pSeDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; - SHashObj* pGroupIdTbNameMap; -} SStreamStateAggOperatorInfo; - typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; uint32_t sortBufSize; // max buffer size for in-memory sort @@ -1064,13 +1063,8 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex); -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex); +void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSessionKey* pKey); bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); @@ -1100,6 +1094,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); +int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); +int32_t buildSessionResultDataBlock(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, + SExprSupp* pSup, SGroupResInfo* pGroupResInfo); int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 1ade4b89e6..55978855d1 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -267,6 +267,8 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData taosThreadMutexInit(&deleter->mutex, NULL); if (NULL == deleter->pDataBlocks) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + destroyDataSinker((SDataSinkHandle*)deleter); + taosMemoryFree(deleter); return TSDB_CODE_QRY_OUT_OF_MEMORY; } *pHandle = deleter; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8f8e486b3a..ffdcf48d48 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -143,9 +143,15 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); - if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { + if (NULL == pBuf) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + + if (!allocBuf(pDispatcher, pInput, pBuf)) { + taosFreeQitem(pBuf); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); @@ -170,6 +176,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); + ASSERT(NULL != pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); @@ -258,6 +265,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->pDataBlocks = taosOpenQueue(); taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { + taosMemoryFree(dispatcher); terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 6a43ea40c6..ed455e5e75 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -81,7 +81,7 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%d", pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); tFreeSSubmitRsp(pInserter->submitRes.pRsp); @@ -323,6 +323,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat int32_t code = tsdbGetTableSchema(inserter->pParam->readHandle->vnode, pInserterNode->tableId, &inserter->pSchema, &suid); if (code) { + destroyDataSinker((SDataSinkHandle*)inserter); return code; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0fadacb2c2..79f6166079 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -122,7 +122,8 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc; - taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); + int32_t size = POINTER_BYTES; + taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), size, fn); } pGroupResInfo->index = 0; @@ -1138,7 +1139,7 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i s.bytes = bytes; s.slotId = slotId; s.precision = precision; - strncpy(s.name, name, tListLen(s.name)); + tstrncpy(s.name, name, tListLen(s.name)); return s; } @@ -1201,14 +1202,21 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; + tExprNode* pExprNode = pExp->pExpr; + + pExprNode->_function.functionId = pFuncNode->funcId; + pExprNode->_function.pFunctNode = pFuncNode; + + tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); #if 1 // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + const char* name = "tbname"; + int32_t len = strlen(name); + + if (!pFuncNode->pParameterList && (memcmp(pExprNode->_function.functionName, name, len) == 0) && + pExprNode->_function.functionName[len] == 0) { + pFuncNode->pParameterList = nodesMakeList(); ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); @@ -1359,7 +1367,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - strncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); + tstrncpy(pCtx->udfName, udfName, TSDB_FUNC_NAME_LEN); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index e527f7e7c1..0d861c1fb9 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -276,6 +276,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, &keyInfo.groupId); if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(keyBuf); return code; } } @@ -294,7 +295,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo if (!exists) { #endif taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); - taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(*uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); } /*}*/ @@ -616,7 +617,7 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { int32_t nOptrWithVal = 0; int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { taosMemoryFreeClear(*pOutput); *len = 0; } @@ -701,10 +702,10 @@ int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { return 0; } -int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo mtInfo) { +int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { memset(pCond, 0, sizeof(SQueryTableDataCond)); pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = mtInfo.schema->nCols; + pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); if (pCond->colList == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -712,15 +713,15 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s } pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = mtInfo.suid; + pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = mtInfo.schema->pSchema[i].type; - pCond->colList[i].bytes = mtInfo.schema->pSchema[i].bytes; - pCond->colList[i].colId = mtInfo.schema->pSchema[i].colId; + pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; + pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; + pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; } return TSDB_CODE_SUCCESS; @@ -844,7 +845,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT taosArrayDestroy(pTaskInfo->tableqinfoList.pTableList); if (mtInfo.uid == 0) return 0; // no data - initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, mtInfo); + initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); pTaskInfo->streamInfo.tableCond.twindows.skey = pOffset->ts; pTaskInfo->tableqinfoList.pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &(STableKeyInfo){.uid = mtInfo.uid, .groupId = 0}); @@ -856,15 +857,15 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; - qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %ld", mtInfo.uid, pOffset->ts); + qDebug("tmqsnap qStreamPrepareScan snapshot data uid %ld ts %"PRId64, mtInfo.uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_META) { SStreamRawScanInfo* pInfo = pOperator->info; SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { - qError("setForSnapShot error. uid:%" PRIi64 " ,version:%" PRIi64, pOffset->uid); + qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); return -1; } - qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %ld", pOffset->uid); + qDebug("tmqsnap qStreamPrepareScan snapshot meta uid %ld ts %"PRId64, pOffset->uid, pOffset->ts); } else if (pOffset->type == TMQ_OFFSET__LOG) { SStreamRawScanInfo* pInfo = pOperator->info; tsdbReaderClose(pInfo->dataReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ea01b961b1..641bcb0c5b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -97,11 +97,10 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator); void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; + ASSERT(pOperator->pTaskInfo != NULL); pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0; - if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - } + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { @@ -895,33 +894,6 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } #endif -static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { -#if 0 - SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; - uint32_t status = BLK_DATA_NOT_LOAD; - - int32_t numOfOutput = 0; // pTableScanInfo->numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - int32_t functionId = pCtx[i].functionId; - int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; - - // group by + first/last should not apply the first/last block filter - if (functionId < 0) { - status |= BLK_DATA_DATA_LOAD; - return status; - } else { - // status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); - // if ((status & BLK_DATA_DATA_LOAD) == BLK_DATA_DATA_LOAD) { - // return status; - // } - } - } - - return status; -#endif - return 0; -} - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1119,13 +1091,13 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM } SFilterInfo* filter = pFilterInfo; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); -// pError("start filter"); + // pError("start filter"); // todo move to the initialization function int32_t code = 0; - bool needFree = false; + bool needFree = false; if (filter == NULL) { needFree = true; code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); @@ -1802,7 +1774,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } else { taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; - qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); + qDebug("%s fetch rsp received, index:%d, code:%s", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -1857,6 +1829,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); if (NULL == pMsg) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); return pTaskInfo->code; } @@ -1874,6 +1847,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); + taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return pTaskInfo->code; @@ -2815,6 +2789,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) { blockDataUpdateTsWindow(pBlock, pInfo->primarySrcSlotId); blockDataCleanup(pInfo->pRes); + blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows); + blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows); doApplyScalarCalculation(pOperator, pBlock, order, scanFlag); if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) { @@ -3118,8 +3094,12 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* } return pOperator; + _error: - destroyAggOperatorInfo(pInfo); + if (pInfo != NULL) { + destroyAggOperatorInfo(pInfo); + } + taosMemoryFreeClear(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -3266,13 +3246,16 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); - SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); + pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); + pOperator->exprSupp.pExprInfo = pExprInfo; + pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode); if (code != TSDB_CODE_SUCCESS) { goto _error; } + SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval @@ -3282,16 +3265,20 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* int32_t type = convertFillType(pPhyFillNode->mode); SResultInfo* pResultInfo = &pOperator->resultInfo; + initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); - initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + code = initExprSupp(&pOperator->exprSupp, pExprInfo, pInfo->numOfExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pInfo->primaryTsCol = ((STargetNode*)pPhyFillNode->pWStartTs)->slotId; pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId; int32_t numOfOutputCols = 0; - SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, - &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + pInfo->pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, + &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, @@ -3300,17 +3287,14 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* goto _error; } - pInfo->pRes = pResBlock; - pInfo->pFinalRes = createOneDataBlock(pResBlock, false); + pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false); blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity); pInfo->pCondition = pPhyFillNode->node.pConditions; - pInfo->pColMatchColInfo = pColMatchColInfo; pOperator->name = "FillOperator"; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_FILL; - pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -3322,8 +3306,11 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* return pOperator; _error: + if (pInfo != NULL) { + destroyFillOperatorInfo(pInfo); + } + taosMemoryFreeClear(pOperator); - taosMemoryFreeClear(pInfo); return NULL; } @@ -3396,8 +3383,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } // this the tags and pseudo function columns, we only keep the tag columns @@ -3411,8 +3398,8 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; pSchema->colId = pColNode->colId; pSchema->type = pColNode->node.resType.type; - pSchema->type = pColNode->node.resType.bytes; - strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } } @@ -4205,42 +4192,6 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf return TSDB_CODE_SUCCESS; } -int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size) { - pSup->currentPageId = -1; - pSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pSup->keySize = sizeof(int64_t) + sizeof(TSKEY); - pSup->pKeyBuf = taosMemoryCalloc(1, pSup->keySize); - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pSup->pResultRows = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - if (pSup->pKeyBuf == NULL || pSup->pResultRows == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pSup->valueSize = size; - - pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); - int32_t pageSize = 4096; - while (pageSize < pSup->resultRowSize * 4) { - pageSize <<= 1u; - } - // at least four pages need to be in buffer - int32_t bufSize = 4096 * 256; - if (bufSize <= pageSize) { - bufSize = pageSize * 4; - } - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s", terrstr(terrno)); - return terrno; - } - int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, pKey, tsTempDir); - for (int32_t i = 0; i < numOfOutput; ++i) { - pCtx[i].saveHandle.pBuf = pSup->pResultBuf; - } - - return code; -} - int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) { SWinKey key = { @@ -4250,7 +4201,6 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul char* value = NULL; int32_t size = pAggSup->resultRowSize; - tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -4355,3 +4305,82 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } + +int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size) { + streamStateSessionPut(pState, key, (const void*)buf, size); + releaseOutputBuf(pState, NULL, (SResultRow*)buf); + return TSDB_CODE_SUCCESS; +} + +int32_t buildSessionResultDataBlock(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, + SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SSessionKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + int32_t code = streamStateSessionGet(pState, pKey, &pVal, &size); + ASSERT(code == 0); + SResultRow* pRow = (SResultRow*)pVal; + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + releaseOutputBuf(pState, NULL, pRow); + continue; + } + + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pKey->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pKey->groupId) { + releaseOutputBuf(pState, NULL, pRow); + break; + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + releaseOutputBuf(pState, NULL, pRow); + break; + } + + pGroupResInfo->index += 1; + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code1 = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code1)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code1)); + T_LONG_JMP(pTaskInfo->env, code1); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstart, top(k, 20) + // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + + pBlock->info.rows += pRow->numOfRows; + // saveSessionDiscBuf(pState, pKey, pVal, size); + releaseOutputBuf(pState, NULL, pRow); + } + blockDataUpdateTsWindow(pBlock, 0); + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index fe6ee4129a..ddb6d73aab 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -384,6 +384,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy numOfRows = TWOMB / pResBlock->info.rowSize; } + initBasicInfo(&pInfo->binfo, pResBlock); initResultSizeInfo(&pOperator->resultInfo, numOfRows); int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str); @@ -391,7 +392,6 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); setFunctionResultOutput(pOperator, &pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, numOfExpr); pInfo->binfo.pRes = pResBlock; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c0c38e2b30..7385e9bb92 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -284,6 +284,19 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } + +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + SExprSupp* pSup = &pTableScanInfo->pseudoSup; + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + } +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -312,8 +325,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->skipBlocks += 1; + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { pCost->loadBlockStatis += 1; @@ -322,6 +336,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -371,17 +386,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true); - - // currently only the tbname pseudo column - if (pTableScanInfo->pseudoSup.numOfExprs > 0) { - SExprSupp* pSup = &pTableScanInfo->pseudoSup; - - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, - GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); if (pTableScanInfo->pFilterNode != NULL) { int64_t st = taosGetTimestampUs(); @@ -937,8 +942,7 @@ static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) { } static bool isSessionWindow(SStreamScanInfo* pInfo) { - return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } static bool isStateWindow(SStreamScanInfo* pInfo) { @@ -1079,12 +1083,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { - SResultRowInfo dumyInfo; + SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; uint64_t groupId = gpIdCol[*pRowIndex]; + while (1) { if (hasGroup) { (*pRowIndex) += 1; @@ -1148,6 +1153,9 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 pResult->info.rows++; } } + + blockDataDestroy(tmpBlock); + if (pResult->info.rows > 0) { pResult->info.calWin = pInfo->updateWin; return pResult; @@ -1182,23 +1190,22 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int32_t dummy = 0; int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. - SResultWindowInfo* pStartWin = - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); - if (!pStartWin) { + SSessionKey startWin = {0}; + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, &startWin); + if (IS_INVALID_SESSION_WIN_KEY(startWin)) { // window has been closed. continue; } - SResultWindowInfo* pEndWin = - getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy); - ASSERT(pEndWin); - TSKEY ts = INT64_MIN; - colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false); - colDataAppend(pDestEndCol, i, (const char*)&pEndWin->win.ekey, false); + SSessionKey endWin = {0}; + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin); + ASSERT(!IS_INVALID_SESSION_WIN_KEY(endWin)); + colDataAppend(pDestStartCol, i, (const char*)&startWin.win.skey, false); + colDataAppend(pDestEndCol, i, (const char*)&endWin.win.ekey, false); + colDataAppendNULL(pDestUidCol, i); colDataAppend(pDestGpCol, i, (const char*)&groupId, false); colDataAppendNULL(pDestCalStartTsCol, i); @@ -1316,7 +1323,7 @@ static void calBlockTag(SExprSupp* pTagCalSup, SSDataBlock* pBlock, SSDataBlock* blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, pTagCalSup->numOfExprs, NULL); + projectApplyFunctions(pTagCalSup->pExprInfo, pResBlock, pSrcBlock, pTagCalSup->pCtx, 1, NULL); ASSERT(pResBlock->info.rows == 1); // build tagArray @@ -1543,7 +1550,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { tsdbReaderClose(pTSInfo->dataReader); pTSInfo->dataReader = NULL; tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - qDebug("queue scan tsdb over, switch to wal ver %d", pTaskInfo->streamInfo.snapshotVer + 1); + qDebug("queue scan tsdb over, switch to wal ver %"PRId64, pTaskInfo->streamInfo.snapshotVer + 1); if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1) < 0) { return NULL; } @@ -1762,6 +1769,7 @@ FETCH_NEXT_BLOCK: pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; printDataBlock(pDelBlock, "stream scan delete result"); if (pInfo->pDeleteDataRes->info.rows > 0) { + blockDataDestroy(pDelBlock); return pInfo->pDeleteDataRes; } else { goto FETCH_NEXT_BLOCK; @@ -2120,9 +2128,6 @@ static void destroyStreamScanOperatorInfo(void* param) { taosMemoryFree(pStreamScan->pPseudoExpr); } - cleanupExprSupp(&pStreamScan->tbnameCalSup); - cleanupExprSupp(&pStreamScan->tagCalSup); - updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); blockDataDestroy(pStreamScan->pUpdateRes); @@ -2568,16 +2573,27 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SMetaReader smrChildTable = {0}; metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smrChildTable, condTableName); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { metaReaderClear(&smrChildTable); blockDataDestroy(dataBlock); pInfo->loadInfo.totalRows = 0; return NULL; } + SMetaReader smrSuperTable = {0}; metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); metaReaderClear(&smrSuperTable); metaReaderClear(&smrChildTable); @@ -2984,8 +3000,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); - strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - strcpy(pInfo->req.user, pInfo->pUser); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); char* buf1 = taosMemoryCalloc(1, contLen); @@ -3316,6 +3332,11 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags const char* idStr) { int64_t st = taosGetTimestampUs(); + if (pHandle == NULL) { + qError("invalid handle, in creating operator tree", idStr); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { qError("failed to getTableList, code: %s", tstrerror(code)); @@ -3795,6 +3816,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pColList); goto _error; } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index c651529c7a..7ca3de5214 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -398,15 +398,16 @@ int32_t finishSortGroup(SOperatorInfo* pOperator) { SGroupSortOperatorInfo* pInfo = pOperator->info; SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pCurrSortHandle); + pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; pInfo->sortExecInfo.loops += sortExecInfo.loops; pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; - if (pInfo->pCurrSortHandle != NULL) { - tsortDestroySortHandle(pInfo->pCurrSortHandle); - } + + tsortDestroySortHandle(pInfo->pCurrSortHandle); pInfo->pCurrSortHandle = NULL; + return TSDB_CODE_SUCCESS; } @@ -658,7 +659,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData blockDataDestroy(p); - qDebug("%s get sorted block, groupId:%0x" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; } @@ -717,10 +718,9 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SMultiwayMergeOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SMultiwayMergeOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SDataBlockDescNode* pDescNode = pPhyNode->pOutputDataBlockDesc; - SSDataBlock* pResBlock = createResDataBlock(pDescNode); - - int32_t rowSize = pResBlock->info.rowSize; + pInfo->binfo.pRes = createResDataBlock(pDescNode); + int32_t rowSize = pInfo->binfo.pRes->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { goto _error; } @@ -734,7 +734,6 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size initResultSizeInfo(&pOperator->resultInfo, 1024); pInfo->groupSort = pMergePhyNode->groupSort; - pInfo->binfo.pRes = pResBlock; pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; pInfo->pInputBlock = pInputBlock; @@ -761,7 +760,10 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size _error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); + if (pInfo != NULL) { + destroyMultiwayMergeOperatorInfo(pInfo); + } + taosMemoryFree(pOperator); return NULL; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 88af03775c..e199cf8406 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -693,7 +693,7 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; - streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); pFillSup->cur.pRowVal = NULL; taosMemoryFree(pFillSup); @@ -736,7 +736,7 @@ static void resetFillWindow(SResultRowData* pRowData) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { resetFillWindow(&pFillSup->prev); - streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); + releaseOutputBuf(NULL, NULL, (SResultRow*)pFillSup->cur.pRowVal); resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); @@ -1681,7 +1681,11 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pInfo->pCondition = pPhyFillNode->node.pConditions; pInfo->pColMatchColInfo = pColMatchColInfo; - initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->srcRowIndex = 0; pOperator->name = "FillOperator"; @@ -1693,7 +1697,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, NULL, destroyStreamFillOperatorInfo, NULL, NULL, NULL); - int32_t code = appendDownstream(pOperator, &downstream, 1); + code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1702,5 +1706,6 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi _error: destroyStreamFillOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; return NULL; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9c219d2765..db9ebf12c2 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -38,8 +38,6 @@ typedef struct SOpenWindowInfo { uint64_t groupId; } SOpenWindowInfo; -static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); - static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, @@ -828,31 +826,9 @@ int32_t compareResKey(void* pKey, void* data, int32_t index) { return -1; } -static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) { - int32_t size = taosArrayGetSize(pUpdated); - SWinKey data = {.ts = ts, .groupId = groupId}; - int32_t index = binarySearchCom(pUpdated, size, &data, TSDB_ORDER_DESC, compareResKey); - if (index == -1) { - index = 0; - } else { - if (compareResKey(&data, pUpdated, index) > 0) { - index++; - } else { - return TSDB_CODE_SUCCESS; - } - } - - SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (newPos == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - newPos->groupId = groupId; - newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; - *(int64_t*)newPos->key = ts; - if (taosArrayInsert(pUpdated, index, &newPos) == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; +static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) { + winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey; + return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); } static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) { @@ -874,10 +850,6 @@ static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedM return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); } -static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { - return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); -} - static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { @@ -915,8 +887,8 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { - taosArraySort(pDelWins, sWinKeyCmprImpl); - taosArrayRemoveDuplicate(pDelWins, sWinKeyCmprImpl, NULL); + taosArraySort(pDelWins, winKeyCmprImpl); + taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL); int32_t delSize = taosArrayGetSize(pDelWins); if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) { return; @@ -1385,19 +1357,6 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, releaseBufPage(pResultBuf, bufPage); } -static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, - int32_t numOfOutput) { - SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (!p1) { - // window has been closed - return false; - } - doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput); - return true; -} - static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; @@ -1455,72 +1414,6 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa } } -bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) { - size_t bytes = sizeof(TSKEY); - SET_RES_WINDOW_KEY(pAggSup->keyBuf, &ts, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - if (!p1) { - // window has been closed - return false; - } - tSimpleHashRemove(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - return true; -} - -static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SSDataBlock* pBlock, - SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) { - SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* tsStarts = (TSKEY*)pStartCol->pData; - SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* tsEnds = (TSKEY*)pEndCol->pData; - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* groupIds = (uint64_t*)pGroupCol->pData; - int64_t numOfWin = tSimpleHashGetSize(pAggSup->pResultRowHashTable); - for (int32_t i = 0; i < pBlock->info.rows; i++) { - TSKEY startTs = TMAX(tsStarts[i], pTwSup->minTs); - TSKEY endTs = TMIN(tsEnds[i], pTwSup->maxTs); - SResultRowInfo dumyInfo; - dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC); - do { - doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); - SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; - if (pDelWins) { - taosArrayPush(pDelWins, &winRes); - } - if (pUpdatedMap) { - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } while (win.skey <= endTs); - } -} - -static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t numOfOutput, - SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; - SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData; - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; - for (int32_t i = 0; i < pBlock->info.rows; i++) { - SResultRowInfo dumyInfo; - dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.ekey <= endTsCols[i]) { - uint64_t winGpId = pGpDatas[i]; - bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); - if (pUpWins && res) { - SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; - taosArrayPush(pUpWins, &winRes); - } - getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } - } -} - static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; @@ -1724,6 +1617,9 @@ static void destroyStateWindowOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosMemoryFreeClear(pInfo->stateKey.pData); cleanupExprSupp(&pInfo->scalarSup); + colDataDestroy(&pInfo->twAggSup.timeWindowData); + cleanupAggSup(&pInfo->aggSup); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -2748,11 +2644,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi goto _error; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; - SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; if (pStateNode->window.pExprs != NULL) { @@ -2775,6 +2667,10 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); + initResultSizeInfo(&pOperator->resultInfo, 4096); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -2842,14 +2738,13 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pSessionNode->window.node.pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initBasicInfo(&pInfo->binfo, pResBlock); - pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; pInfo->gap = pSessionNode->gap; @@ -2920,11 +2815,12 @@ bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0); } -static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SArray* pWinArray, SHashObj* pUpdatedMap) { +static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t size = taosArrayGetSize(pWinArray); int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SExprSupp* pSup = &pOperator->exprSupp; if (!pInfo->pChildren) { return; } @@ -2932,8 +2828,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SAr SWinKey* pWinRes = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval); - if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup) && - isCloseWindow(&parentWin, &pInfo->twAggSup)) { + if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup)) { continue; } @@ -3220,6 +3115,14 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + SWinKey key = { + .ts = pResult->win.skey, + .groupId = groupId, + }; + tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); + } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); @@ -3335,7 +3238,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL); - rebuildIntervalWindow(pOperator, pSup, delWins, pUpdatedMap); + rebuildIntervalWindow(pOperator, delWins, pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); @@ -3472,6 +3375,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + initBasicInfo(&pInfo->binfo, pResBlock); int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { @@ -3479,7 +3383,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } initStreamFunciton(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - initBasicInfo(&pInfo->binfo, pResBlock); ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -3561,42 +3464,18 @@ _error: } void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { - taosMemoryFreeClear(pSup->pKeyBuf); - void** pIte = NULL; - while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - taosArrayDestroy(pWins); - } - taosHashCleanup(pSup->pResultRows); - destroyDiskbasedBuf(pSup->pResultBuf); - blockDataDestroy(pSup->pScanBlock); -} - -void destroyStateWinInfo(void* ptr) { - if (ptr == NULL) { - return; - } - SStateWindowInfo* pWin = (SStateWindowInfo*)ptr; - taosMemoryFreeClear(pWin->stateKey.pData); -} - -void destroyStateStreamAggSupporter(SStreamAggSupporter* pSup) { - taosMemoryFreeClear(pSup->pKeyBuf); - void** pIte = NULL; - while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - taosArrayDestroyEx(pWins, (FDelete)destroyStateWinInfo); - } - taosHashCleanup(pSup->pResultRows); + tSimpleHashCleanup(pSup->pResultRows); destroyDiskbasedBuf(pSup->pResultBuf); blockDataDestroy(pSup->pScanBlock); + taosMemoryFreeClear(pSup->pState); + taosMemoryFreeClear(pSup->pDummyCtx); } void destroyStreamSessionAggOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); destroyStreamAggSupporter(&pInfo->streamAggSup); - cleanupGroupResInfo(&pInfo->groupResInfo); + if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { @@ -3610,8 +3489,7 @@ void destroyStreamSessionAggOperatorInfo(void* param) { blockDataDestroy(pInfo->pDelRes); blockDataDestroy(pInfo->pWinBlock); blockDataDestroy(pInfo->pUpdateRes); - destroySqlFunctionCtx(pInfo->pDummyCtx, 0); - taosHashCleanup(pInfo->pStDeleted); + tSimpleHashCleanup(pInfo->pStDeleted); taosHashCleanup(pInfo->pGroupIdTbNameMap); taosMemoryFreeClear(param); @@ -3619,14 +3497,13 @@ void destroyStreamSessionAggOperatorInfo(void* param) { int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) { + initBasicInfo(pBasicInfo, pResultBlock); int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); if (code != TSDB_CODE_SUCCESS) { return code; } + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - - initBasicInfo(pBasicInfo, pResultBlock); - for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].saveHandle.pBuf = NULL; } @@ -3641,127 +3518,62 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } } -void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, - uint16_t type, int32_t tsColIndex) { +void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t waterMark, uint16_t type, + int32_t tsColIndex) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { SStreamPartitionOperatorInfo* pScanInfo = downstream->info; pScanInfo->tsColIndex = tsColIndex; } if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex); + initDownStream(downstream->pDownstream[0], pAggSup, waterMark, type, tsColIndex); return; } SStreamScanInfo* pScanInfo = downstream->info; - pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; + pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } -int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, - int32_t numOfOutput) { - return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo)); -} - -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo) { - SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t code = TSDB_CODE_OUT_OF_MEMORY; - SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, int64_t gap, + SStreamState* pState, int32_t keySize, int16_t keyType) { + pSup->resultRowSize = keySize + getResultRowSize(pCtx, numOfOutput); + pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR); + pSup->gap = gap; + pSup->stateKeySize = keySize; + pSup->stateKeyType = keyType; + pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx)); + if (pSup->pDummyCtx == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; } - pOperator->pTaskInfo = pTaskInfo; + initDummyFunction(pSup->pDummyCtx, pCtx, numOfOutput); + pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pSup->pState) = *pState; + streamStateSetNumber(pSup->pState, -1); - initResultSizeInfo(&pOperator->resultInfo, 4096); - if (pSessionNode->window.pExprs != NULL) { - int32_t numOfScalar = 0; - SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); - code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } - SExprSupp* pSup = &pOperator->exprSupp; - - code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); - if (pInfo->pDummyCtx == NULL) { - goto _error; - } - initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); - - pInfo->twAggSup = (STimeWindowAggSupp){ - .waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - }; - - initResultRowInfo(&pInfo->binfo.resultRowInfo); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - - pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; - if (pSessionNode->window.pTsEnd) { - pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId; - } - pInfo->gap = pSessionNode->gap; - pInfo->binfo.pRes = pResBlock; - pInfo->order = TSDB_ORDER_ASC; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); - pInfo->pDelIterator = NULL; - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); - pInfo->pChildren = NULL; - pInfo->isFinal = false; - pInfo->pPhyNode = pPhyNode; - pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pSup->pResultRows = tSimpleHashInit(32, hashFn); - pInfo->pGroupIdTbNameMap = - taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); - - pOperator->name = "StreamSessionWindowAggOperator"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); - if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, - pInfo->primaryTsIndex); - code = appendDownstream(pOperator, &downstream, 1); + int32_t pageSize = 4096; + while (pageSize < pSup->resultRowSize * 4) { + pageSize <<= 1u; } - return pOperator; - -_error: - if (pInfo != NULL) { - destroyStreamSessionAggOperatorInfo(pInfo); + // at least four pages need to be in buffer + int32_t bufSize = 4096 * 256; + if (bufSize <= pageSize) { + bufSize = pageSize * 4; + } + if (!osTempSpaceAvailable()) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + qError("Init stream agg supporter failed since %s", terrstr(terrno)); + return terrno; + } + int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].saveHandle.pBuf = pSup->pResultBuf; } - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - -int64_t getSessionWindowEndkey(void* data, int32_t index) { - SArray* pWinInfos = (SArray*)data; - SResultWindowInfo* pWin = taosArrayGet(pWinInfos, index); - return pWin->win.ekey; + return TSDB_CODE_SUCCESS; } bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { @@ -3771,305 +3583,243 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { return false; } -bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); } - -static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) { - SResultWindowInfo win = { - .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; - return taosArrayInsert(pWinInfos, index, &win); +bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { + return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap); } -static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) { - SResultWindowInfo win = { - .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; - return taosArrayPush(pWinInfos, &win); +void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SSessionKey* pKey) { + pKey->win.skey = startTs; + pKey->win.ekey = endTs; + pKey->groupId = groupId; + SStreamStateCur* pCur = streamStateSessionGetCur(pAggSup->pState, pKey); + int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); + streamStateFreeCur(pCur); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_KEY_INVALID(pKey); + } } -SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { - void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t)); - SArray* pWinInfos = NULL; - if (ite == NULL) { - pWinInfos = taosArrayInit(1024, pAggSup->valueSize); - taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*)); +bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; } + +void setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + SResultWindowInfo* pCurWin) { + pCurWin->sessionWin.groupId = groupId; + pCurWin->sessionWin.win.skey = startTs - pAggSup->gap; + pCurWin->sessionWin.win.ekey = endTs + pAggSup->gap; + int32_t size = pAggSup->resultRowSize; + int32_t code = streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, &pCurWin->pOutputBuf, &size); + if (code == TSDB_CODE_SUCCESS) { + pCurWin->isOutput = true; } else { - pWinInfos = *ite; + pCurWin->sessionWin.win.skey = startTs; + pCurWin->sessionWin.win.ekey = endTs; } - return pWinInfos; } -// don't add new window -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex) { - STimeWindow searchWin = {.skey = startTs, .ekey = endTs}; - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - return NULL; +int32_t getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo) { + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin, (const void**)&pWinInfo->pOutputBuf, &size); + if (code != TSDB_CODE_SUCCESS) { + return code; } - // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); - SResultWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { - *pIndex = index; - return pWin; - } - } - - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { - *pIndex = index + 1; - return pWin; - } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index + 1; - return pWin; - } - } - - return NULL; + streamStateCurNext(pAggSup->pState, pCur); + return TSDB_CODE_SUCCESS; +} +void saveDeleteInfo(SArray* pWins, SSessionKey key) { + // key.win.ekey = key.win.skey; + taosArrayPush(pWins, &key); } -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - int64_t gap, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; +void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) { + key.win.ekey = key.win.skey; + tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0); +} - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - *pIndex = 0; - return addNewSessionWindow(pWinInfos, startTs, endTs); - } - // find the first position which is smaller than the key - int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); - SResultWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap)) { - *pIndex = index; - return pWin; - } - } +static void removeSessionResult(SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey key) { + key.win.ekey = key.win.skey; + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); + tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); +} - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap)) { - *pIndex = index + 1; - return pWin; - } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { - *pIndex = index; - return pWin; - } +static void removeSessionResults(SSHashObj* pHashMap, SArray* pWins) { + if (tSimpleHashGetSize(pHashMap) == 0) { + return; } - - if (index == size - 1) { - *pIndex = taosArrayGetSize(pWinInfos); - return addNewSessionWindow(pWinInfos, startTs, endTs); + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + SSessionKey* pWin = taosArrayGet(pWins, i); + if (!pWin) continue; + SSessionKey key = *pWin; + key.win.ekey = key.win.skey; + tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey)); } - *pIndex = index + 1; - return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1); } int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId, - int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) { + int32_t rows, int32_t start, int64_t gap, SSHashObj* pResultRows, SSHashObj* pStUpdated, + SSHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { return i - start; } - if (pWinInfo->win.skey > pStartTs[i]) { + if (pWinInfo->sessionWin.win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->isOutput = false; + saveDeleteRes(pStDeleted, pWinInfo->sessionWin); } - pWinInfo->win.skey = pStartTs[i]; + removeSessionResult(pStUpdated, pResultRows, pWinInfo->sessionWin); + pWinInfo->sessionWin.win.skey = pStartTs[i]; } - pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pStartTs[i]); + pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]); if (pEndTs) { - pWinInfo->win.ekey = TMAX(pWinInfo->win.ekey, pEndTs[i]); + pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]); } } return rows - start; } -static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, - uint64_t groupId, int32_t numOfOutput, int32_t* rowEntryInfoOffset, - SStreamAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { - assert(pWinInfo->win.skey <= pWinInfo->win.ekey); - // too many time window in query - int32_t size = taosArrayGetSize(pAggSup->pCurWins); - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && size > MAX_INTERVAL_TIME_WINDOW) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); - } - - if (pWinInfo->pos.pageId == -1) { - *pResult = getNewResultRow(pAggSup->pResultBuf, &pAggSup->currentPageId, pAggSup->resultRowSize); - if (*pResult == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - // add a new result set for a new group - pWinInfo->pos.pageId = (*pResult)->pageId; - pWinInfo->pos.offset = (*pResult)->offset; - } else { - *pResult = getResultRowByPos(pAggSup->pResultBuf, &pWinInfo->pos, true); - if (!(*pResult)) { - qError("getResultRowByPos return NULL, TID:%s", GET_TASKID(pTaskInfo)); - return TSDB_CODE_FAILED; - } - } - +static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset) { + ASSERT(pWinInfo->sessionWin.win.skey <= pWinInfo->sessionWin.win.ekey); + *pResult = (SResultRow*)pWinInfo->pOutputBuf; // set time window for current result - (*pResult)->win = pWinInfo->win; + (*pResult)->win = pWinInfo->sessionWin.win; setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStreamAggSupporter* pAggSup, - SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, - int32_t numOutput, SOperatorInfo* pOperator) { +static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId); - TSKEY* tsCols = (int64_t*)pColDataInfo->pData; - int32_t code = setWindowOutputBuf(pCurWin, pResult, pSup->pCtx, pSDataBlock->info.groupId, numOutput, - pSup->rowEntryInfoOffset, pAggSup, pTaskInfo); + int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } - updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false); - doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, pSDataBlock->info.rows, numOutput); - SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId); - setBufPageDirty(bufPage, true); - releaseBufPage(pAggSup->pResultBuf, bufPage); + updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); + doApplyFunctions(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, - int32_t numOutput, SOperatorInfo* pOperator) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, - pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); +static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) { + streamStateSessionDel(pAggSup->pState, pKey); + tSimpleHashRemove(pAggSup->pResultRows, pKey, sizeof(SSessionKey)); + return true; } -static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, - SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, - int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) { - return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, - pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); -} - -int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) { - SResultWindowInfo* pCurWin = taosArrayGet(pWinInfos, startIndex); - int32_t size = taosArrayGetSize(pWinInfos); - // Just look for the window behind StartIndex - for (int32_t i = startIndex + 1; i < size; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pWinInfos, i); - if (!isInWindow(pCurWin, pWinInfo->win.skey, gap)) { - return i - startIndex - 1; - } +static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { + void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey)); + if (pVal) { + SResultWindowInfo* pWin = pVal; + pWinInfo->isOutput = pWin->isOutput; } - - return size - startIndex - 1; + return TSDB_CODE_SUCCESS; } -void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId, - int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pCurWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - num += startIndex + 1; - ASSERT(num <= taosArrayGetSize(pInfo->streamAggSup.pCurWins)); - // Just look for the window behind StartIndex - for (int32_t i = startIndex + 1; i < num; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pInfo->streamAggSup.pCurWins, i); - SResultRow* pWinResult = NULL; - setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); - compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); - if (pWinInfo->isOutput && pStDeleted) { - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->isOutput = false; - } - taosArrayRemove(pInfo->streamAggSup.pCurWins, i); - SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId); - releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage); +SStreamStateCur* getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, + SResultWindowInfo* pNextWin) { + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); + pNextWin->isOutput = true; + setSessionWinOutputInfo(pStUpdated, pNextWin); + int32_t size = 0; + pNextWin->sessionWin = pCurWin->sessionWin; + int32_t code = streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin, (const void**)&pNextWin->pOutputBuf, &size); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_INVALID(*pNextWin); } - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId); - ASSERT(num > 0); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + return pCur; } -static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, - SHashObj* pStDeleted, bool hasEndTs) { +static void compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, + SSHashObj* pStDeleted) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + // Just look for the window behind StartIndex + while (1) { + SResultWindowInfo winInfo = {0}; + SStreamStateCur* pCur = getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo); + if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap)) { + streamStateFreeCur(pCur); + break; + } + SResultRow* pWinResult = NULL; + initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, true); + compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey)); + if (winInfo.isOutput && pStDeleted) { + saveDeleteRes(pStDeleted, winInfo.sessionWin); + } + removeSessionResult(pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); + doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); + streamStateFreeCur(pCur); + } +} + +int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) { + saveSessionDiscBuf(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pOutputBuf, pAggSup->resultRowSize); + return TSDB_CODE_SUCCESS; +} + +static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; uint64_t groupId = pSDataBlock->info.groupId; - int64_t gap = pInfo->gap; int64_t code = TSDB_CODE_SUCCESS; + SResultRow* pResult = NULL; + int32_t rows = pSDataBlock->info.rows; + int32_t winRows = 0; - int32_t step = 1; - bool ascScan = true; - TSKEY* startTsCols = NULL; - TSKEY* endTsCols = NULL; - SResultRow* pResult = NULL; - int32_t winRows = 0; - - ASSERT(pSDataBlock->pDataBlock); SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - startTsCols = (int64_t*)pStartTsCol->pData; + TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; if (hasEndTs) { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex); } else { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); } - endTsCols = (int64_t*)pEndTsCol->pData; + TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - for (int32_t i = 0; i < pSDataBlock->info.rows;) { + for (int32_t i = 0; i < rows;) { if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) { i++; continue; } - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex); - winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, groupId, pSDataBlock->info.rows, i, pInfo->gap, - pStDeleted); - code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator); + SResultWindowInfo winInfo = {0}; + setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo); + setSessionWinOutputInfo(pStUpdated, &winInfo); + winRows = updateSessionWindowInfo(&winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap, + pAggSup->pResultRows, pStUpdated, pStDeleted); + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, + pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); + saveSessionOutputBuf(pAggSup, &winInfo); - int32_t winNum = getNumCompactWindow(pAggSup->pCurWins, winIndex, gap); - if (winNum > 0) { - compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, pStDeleted, pOperator); - } - pCurWin->isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { - SWinKey value = {.ts = pCurWin->win.skey, .groupId = groupId}; - code = taosHashPut(pStUpdated, &pCurWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); + code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->isOutput = true; } + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + SSessionKey key = winInfo.sessionWin; + key.win.ekey = key.win.skey; + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); + } + i += winRows; } } @@ -4083,8 +3833,7 @@ void deleteWindow(SArray* pWinInfos, int32_t index, FDelete fp) { taosArrayRemove(pWinInfos, index); } -static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, int64_t gap, SArray* result, - FDelete fp) { +static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -4092,85 +3841,70 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); - if (!pCurWin) { - continue; - } - - do { - SResultWindowInfo delWin = *pCurWin; - deleteWindow(pAggSup->pCurWins, winIndex, fp); - if (result) { - delWin.groupId = gpDatas[i]; - taosArrayPush(result, &delWin); - } - if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) { + while (1) { + SSessionKey curWin = {0}; + getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin); + if (IS_INVALID_SESSION_WIN_KEY(curWin)) { break; } - pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex); - } while (pCurWin->win.skey <= endDatas[i]); - } -} - -static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex, - int32_t numOfOutput, int64_t gap, SArray* result) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - SColumnInfoData* pGpDataInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* gpCols = (uint64_t*)pGpDataInfo->pData; - int32_t step = 0; - for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, tsCols[i], INT64_MIN, gpCols[i], gap, &winIndex); - if (!pCurWin || pCurWin->pos.pageId == -1) { - // window has been closed. - step = 1; - continue; - } - step = updateSessionWindowInfo(pCurWin, tsCols, NULL, 0, pBlock->info.rows, i, gap, NULL); - ASSERT(isInWindow(pCurWin, tsCols[i], gap)); - doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); - if (result) { - pCurWin->groupId = gpCols[i]; - taosArrayPush(result, pCurWin); + doDeleteSessionWindow(pAggSup, &curWin); + if (result) { + saveDeleteInfo(result, curWin); + } } } } -static int32_t copyUpdateResult(SHashObj* pStUpdated, SArray* pUpdated) { - void* pData = NULL; - size_t keyLen = 0; - while ((pData = taosHashIterate(pStUpdated, pData)) != NULL) { - void* key = taosHashGetKey(pData, &keyLen); - ASSERT(keyLen == sizeof(SResultRowPosition)); - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (pos == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - pos->groupId = ((SWinKey*)pData)->groupId; - pos->pos = *(SResultRowPosition*)key; - *(int64_t*)pos->key = ((SWinKey*)pData)->ts; - taosArrayPush(pUpdated, &pos); +static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { + SSessionKey* pWin1 = (SSessionKey*)pKey1; + SSessionKey* pWin2 = (SSessionKey*)pKey2; + + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; } - taosArraySort(pUpdated, resultrowComparAsc); + + if (pWin1->win.skey > pWin2->win.skey) { + return 1; + } else if (pWin1->win.skey < pWin2->win.skey) { + return -1; + } + + return 0; +} + +static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pIte, &keyLen); + ASSERT(keyLen == sizeof(SSessionKey)); + taosArrayPush(pUpdated, key); + } + taosArraySort(pUpdated, sessionKeyCompareAsc); return TSDB_CODE_SUCCESS; } -void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { +void doBuildDeleteDataBlock(SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite) { blockDataCleanup(pBlock); - int32_t size = taosHashGetSize(pStDeleted); + int32_t size = tSimpleHashGetSize(pStDeleted); if (size == 0) { return; } blockDataEnsureCapacity(pBlock, size); - size_t keyLen = 0; - while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { - SWinKey* res = *Ite; + size_t keyLen = 0; + int32_t iter = 0; + while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) { + if (pBlock->info.rows + 1 > pBlock->info.capacity) { + break; + } + SSessionKey* res = tSimpleHashGetKey(*Ite, &keyLen); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false); + colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false); + colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); colDataAppendNULL(pUidCol, pBlock->info.rows); SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); @@ -4180,167 +3914,137 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); colDataAppendNULL(pCalEdCol, pBlock->info.rows); pBlock->info.rows += 1; - if (pBlock->info.rows + 1 >= pBlock->info.capacity) { - break; - } } if ((*Ite) == NULL) { - taosHashClear(pStDeleted); + tSimpleHashClear(pStDeleted); } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, - SOperatorInfo* pOperator, SHashObj* pStUpdated) { - SExprSupp* pSup = &pOperator->exprSupp; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int32_t size = taosArrayGetSize(pWinArray); +static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t size = taosArrayGetSize(pWinArray); + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + int32_t numOfOutput = pSup->numOfExprs; + int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - uint64_t groupId = pParentWin->groupId; - int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); + SSessionKey* pWinKey = taosArrayGet(pWinArray, i); + int32_t num = 0; + SResultWindowInfo parentWin = {0}; for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; - SArray* pChWins = getWinInfos(&pChInfo->streamAggSup, groupId); - int32_t chWinSize = taosArrayGetSize(pChWins); - int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); - if (index < 0) { - index = 0; - } - for (int32_t k = index; k < chWinSize; k++) { - SResultWindowInfo* pChWin = taosArrayGet(pChWins, k); - if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) { - int32_t winIndex = 0; - SResultWindowInfo* pNewParWin = - getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex); - SResultRow* pPareResult = NULL; - setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - SResultRow* pChResult = NULL; - setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, - pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true); - compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - - int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap); - if (winNum > 0) { - compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator); + SStreamAggSupporter* pChAggSup = &pChInfo->streamAggSup; + SStreamStateCur* pCur = streamStateSessionGetCur(pChAggSup->pState, pWinKey); + SResultRow* pResult = NULL; + SResultRow* pChResult = NULL; + while (1) { + SResultWindowInfo childWin = {0}; + childWin.sessionWin = *pWinKey; + int32_t code = getSessionWinBuf(pChAggSup, pCur, &childWin); + if (code == TSDB_CODE_SUCCESS && pWinKey->win.skey <= childWin.sessionWin.win.skey && + childWin.sessionWin.win.ekey <= pWinKey->win.ekey) { + if (num == 0) { + setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin); + code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + break; + } } - - SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); - releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); - - bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); - SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId}; - taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); - } else if (!pChWin->isClosed) { + num++; + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, true); + initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput, + pChild->exprSupp.rowEntryInfoOffset); + compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL); + saveResult(parentWin, pStUpdated); + } else { break; } } + streamStateFreeCur(pCur); + } + if (num > 0) { + saveSessionOutputBuf(pAggSup, &parentWin); } } } -typedef SResultWindowInfo* (*__get_win_info_)(void*); -SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; } -SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } - -int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn, - bool delete, FDelete fp) { - // Todo(liuyao) save window to tdb - void** pIte = NULL; - size_t keyLen = 0; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen); - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (isCloseWindow(&pSeWin->win, pTwSup)) { - if (!pSeWin->isClosed) { - pSeWin->isClosed = true; - if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, *pGroupId, pClosed); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - pSeWin->isOutput = true; - } - if (delete) { - deleteWindow(pWins, i, fp); - i--; - size = taosArrayGetSize(pWins); - } +int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) { + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + SResultWindowInfo* pWinInfo = pIte; + if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) { + if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) { + int32_t code = saveResult(*pWinInfo, pClosed); + if (code != TSDB_CODE_SUCCESS) { + return code; } - continue; } - break; + tSimpleHashIterateRemove(pHashMap, &pWinInfo->sessionWin, sizeof(SSessionKey), &pIte, &iter); } } return TSDB_CODE_SUCCESS; } -static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs, bool delete, FDelete fp) { +static void closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL, getResWinForSession, delete, fp); + closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL); } } -int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) { - void** pIte = NULL; - while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - void* pWin = taosArrayGet(pWins, i); - SResultWindowInfo* pSeWin = fn(pWin); - if (!pSeWin->isClosed) { - int32_t code = saveResult(pSeWin->win.skey, pSeWin->pos.pageId, pSeWin->pos.offset, 0, pClosed); - pSeWin->isOutput = true; - } - } +int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { + SResultWindowInfo* pWinInfo = *(void**)pIte; + saveResult(*pWinInfo, pStUpdated); } return TSDB_CODE_SUCCESS; } -static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) { +static void copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) { int32_t size = taosArrayGetSize(pResWins); for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i); - SWinKey res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId}; - taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); + SSessionKey* pWinKey = taosArrayGet(pResWins, i); + if (!pWinKey) continue; + SSessionKey winInfo = *pWinKey; + winInfo.win.ekey = winInfo.win.skey; + tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0); } } -static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) { - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWin = taosArrayGet(pWins, i); - taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition)); - } +void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { + pGroupResInfo->pRows = pArrayList; + pGroupResInfo->index = 0; + ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } -static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) { - int32_t size = taosHashGetSize(pStDeleted); - if (size == 0) { +void doBuildSessionResult(SOperatorInfo* pOperator, SStreamState* pState, SGroupResInfo* pGroupResInfo, + SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; return; } - int32_t num = taosArrayGetSize(update); - for (int32_t i = 0; i < num; i++) { - SResKeyPos* pos = taosArrayGetP(update, i); - SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId}; - taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey)); - } + // clear the existed group id + pBlock->info.groupId = 0; + buildSessionResultDataBlock(pTaskInfo, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); } static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { @@ -4348,6 +4052,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { @@ -4356,18 +4061,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); + return pBInfo->pRes; } - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doSetOperatorCompleted(pOperator); + return NULL; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); // SResKeyPos while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4381,38 +4088,25 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ } - if (pBlock->info.type == STREAM_CLEAR) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, START_TS_COLUMN_INDEX, - pOperator->exprSupp.numOfExprs, 0, pWins); - if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; - doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, - pChildOp->exprSupp.numOfExprs, 0, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); - } - taosArrayDestroy(pWins); - continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { + SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL); + doDeleteTimeWindows(pAggSup, pBlock, pWins); + removeSessionResults(pStUpdated, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 - doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL); - rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); + doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); + rebuildSessionWindow(pOperator, pWins, pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); - removeSessionResults(pStUpdated, pWins); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); + getAllSessionWindow(pAggSup->pResultRows, pStUpdated); continue; } @@ -4420,7 +4114,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pExprSup = &pInfo->scalarSupp; projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } - // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); if (IS_FINAL_OP(pInfo)) { @@ -4447,39 +4140,121 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession, - pInfo->ignoreExpiredData, NULL); - closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL); + closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated); + closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); copyUpdateResult(pStUpdated, pUpdated); - removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); - taosHashCleanup(pStUpdated); - - finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + removeSessionResults(pInfo->pStDeleted, pUpdated); + tSimpleHashCleanup(pStUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); + return pBInfo->pRes; + } + + doSetOperatorCompleted(pOperator); + return NULL; +} + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + int32_t numOfCols = 0; + int32_t code = TSDB_CODE_OUT_OF_MEMORY; + SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pOperator->pTaskInfo = pTaskInfo; + + initResultSizeInfo(&pOperator->resultInfo, 4096); + if (pSessionNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + SExprSupp* pSup = &pOperator->exprSupp; + + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, pSessionNode->gap, + pTaskInfo->streamInfo.pState, 0, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + }; + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId; + if (pSessionNode->window.pTsEnd) { + pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId; + } + pInfo->binfo.pRes = pResBlock; + pInfo->order = TSDB_ORDER_ASC; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStDeleted = tSimpleHashInit(64, hashFn); + pInfo->pDelIterator = NULL; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + pInfo->pChildren = NULL; + pInfo->isFinal = false; + pInfo->pPhyNode = pPhyNode; + pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pGroupIdTbNameMap = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + + pOperator->name = "StreamSessionWindowAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); + if (downstream) { + initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); + code = appendDownstream(pOperator, &downstream, 1); + } + return pOperator; + +_error: + if (pInfo != NULL) { + destroyStreamSessionAggOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; } static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { - void** pIte = NULL; - while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) { - SArray* pWins = (SArray*)(*pIte); - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i); - pWin->pos.pageId = -1; - pWin->pos.offset = -1; - } - } - clearDiskbasedBuf(pInfo->streamAggSup.pResultBuf); - pInfo->streamAggSup.currentPageId = -1; + tSimpleHashClear(pInfo->streamAggSup.pResultRows); + streamStateSessionClear(pInfo->streamAggSup.pState); } static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { @@ -4487,13 +4262,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; } { - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4501,30 +4277,23 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "semi session"); + printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { - pInfo->returnUpdate = false; - // process the rest of the data - printDataBlock(pInfo->pUpdateRes, "semi session"); - return pInfo->pUpdateRes; - } - if (pOperator->status == OP_RES_TO_RETURN) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamSessionOperator(pInfo); - pOperator->status = OP_EXEC_DONE; + doSetOperatorCompleted(pOperator); return NULL; } } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4540,24 +4309,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { /*printf("\n\n put tbname %s\n\n", pBlock->info.parTbName);*/ } - if (pBlock->info.type == STREAM_CLEAR) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, START_TS_COLUMN_INDEX, pSup->numOfExprs, 0, pWins); - removeSessionResults(pStUpdated, pWins); - taosArrayDestroy(pWins); - copyDataBlock(pInfo->pUpdateRes, pBlock); - pInfo->returnUpdate = true; - break; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { // gap must be 0 SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL); - copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); removeSessionResults(pStUpdated, pWins); + copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated); continue; } @@ -4575,15 +4337,13 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; copyUpdateResult(pStUpdated, pUpdated); - removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); - taosHashCleanup(pStUpdated); + removeSessionResults(pInfo->pStDeleted, pUpdated); + tSimpleHashCleanup(pStUpdated); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; @@ -4591,18 +4351,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "semi session"); + printDataBlock(pInfo->pDelRes, "semi session delete"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { - pInfo->returnUpdate = false; - // process the rest of the data - printDataBlock(pInfo->pUpdateRes, "semi session"); - return pInfo->pUpdateRes; - } - - pOperator->status = OP_EXEC_DONE; + clearFunctionContext(&pOperator->exprSupp); + // semi interval operator clear disk buffer + clearStreamSessionOperator(pInfo); + doSetOperatorCompleted(pOperator); return NULL; } @@ -4635,11 +4391,14 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); - if (pChild == NULL) { + SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); + if (pChildOp == NULL) { goto _error; } - taosArrayPush(pInfo->pChildren, &pChild); + SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info; + pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; + streamStateSetNumber(pChInfo->streamAggSup.pState, i); + taosArrayPush(pInfo->pChildren, &pChildOp); } } return pOperator; @@ -4648,7 +4407,6 @@ _error: if (pInfo != NULL) { destroyStreamSessionAggOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -4657,7 +4415,7 @@ _error: void destroyStreamStateOperatorInfo(void* param) { SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); - destroyStateStreamAggSupporter(&pInfo->streamAggSup); + destroyStreamAggSupporter(&pInfo->streamAggSup); cleanupGroupResInfo(&pInfo->groupResInfo); if (pInfo->pChildren != NULL) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -4666,156 +4424,79 @@ void destroyStreamStateOperatorInfo(void* param) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); blockDataDestroy(pInfo->pDelRes); - taosHashCleanup(pInfo->pSeDeleted); + tSimpleHashCleanup(pInfo->pSeDeleted); taosHashCleanup(pInfo->pGroupIdTbNameMap); - destroySqlFunctionCtx(pInfo->pDummyCtx, 0); - taosMemoryFreeClear(param); } -int64_t getStateWinTsKey(void* data, int32_t index) { - SStateWindowInfo* pStateWin = taosArrayGet(data, index); - return pStateWin->winInfo.win.ekey; -} - -SStateWindowInfo* addNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, SColumn* pCol) { - SStateWindowInfo win = { - .stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, - .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, - .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; - if (IS_VAR_DATA_TYPE(win.stateKey.type)) { - varDataCopy(win.stateKey.pData, pKeyData); - } else { - memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); - } - return taosArrayPush(pWinInfos, &win); -} - -SStateWindowInfo* insertNewStateWindow(SArray* pWinInfos, TSKEY ts, char* pKeyData, int32_t index, SColumn* pCol) { - SStateWindowInfo win = { - .stateKey.bytes = pCol->bytes, - .stateKey.type = pCol->type, - .stateKey.pData = taosMemoryCalloc(1, pCol->bytes), - .winInfo.pos.offset = -1, - .winInfo.pos.pageId = -1, - .winInfo.win.skey = ts, - .winInfo.win.ekey = ts, - .winInfo.isOutput = false, - .winInfo.isClosed = false, - }; - if (IS_VAR_DATA_TYPE(win.stateKey.type)) { - varDataCopy(win.stateKey.pData, pKeyData); - } else { - memcpy(win.stateKey.pData, pKeyData, win.stateKey.bytes); - } - return taosArrayInsert(pWinInfos, index, &win); -} - bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) { - if (pWin->winInfo.win.skey <= ts && ts <= pWin->winInfo.win.ekey) { + if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) { return true; } return false; } bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) { - return pKeyData && compareVal(pKeyData, &pWin->stateKey); + return pKeyData && compareVal(pKeyData, pWin->pStateKey); } -SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - int32_t size = taosArrayGetSize(pWinInfos); - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); - SStateWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isTsInWindow(pWin, ts)) { - *pIndex = index; - return pWin; - } - } - - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isTsInWindow(pWin, ts)) { - *pIndex = index + 1; - return pWin; - } - } - *pIndex = 0; - return NULL; +bool compareStateKey(void* data, void* key) { + SStateKeys* stateKey = (SStateKeys*)key; + stateKey->pData = (char*)key + sizeof(SStateKeys); + return compareVal(data, stateKey); } -SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, - SColumn* pCol, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); - pAggSup->pCurWins = pWinInfos; - int32_t size = taosArrayGetSize(pWinInfos); - if (size == 0) { - *pIndex = 0; - return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); - } - int32_t index = binarySearch(pWinInfos, size, ts, TSDB_ORDER_DESC, getStateWinTsKey); - SStateWindowInfo* pWin = NULL; - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isTsInWindow(pWin, ts)) { - *pIndex = index; - return pWin; +void setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, + SStateWindowInfo* pCurWin, SStateWindowInfo* pNextWin) { + int32_t size = pAggSup->resultRowSize; + pCurWin->winInfo.sessionWin.groupId = groupId; + pCurWin->winInfo.sessionWin.win.skey = ts; + pCurWin->winInfo.sessionWin.win.ekey = ts; + int32_t code = + streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData, pAggSup->stateKeySize, + compareStateKey, &pCurWin->winInfo.pOutputBuf, &size); + pCurWin->pStateKey = + (SStateKeys*)((char*)pCurWin->winInfo.pOutputBuf + (pAggSup->resultRowSize - pAggSup->stateKeySize)); + pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys); + pCurWin->pStateKey->type = pAggSup->stateKeyType; + pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys); + pCurWin->pStateKey->isNull = false; + + if (code == TSDB_CODE_SUCCESS) { + pCurWin->winInfo.isOutput = true; + } else { + if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) { + varDataCopy(pCurWin->pStateKey->pData, pKeyData); + } else { + memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes); } } - if (index + 1 < size) { - pWin = taosArrayGet(pWinInfos, index + 1); - if (isTsInWindow(pWin, ts) || isEqualStateKey(pWin, pKeyData)) { - *pIndex = index + 1; - return pWin; - } + pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin; + pNextWin->winInfo.pOutputBuf = NULL; + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); + code = streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_INVALID(pNextWin->winInfo); } - - if (index >= 0) { - pWin = taosArrayGet(pWinInfos, index); - if (isEqualStateKey(pWin, pKeyData)) { - *pIndex = index; - return pWin; - } - } - - if (index == size - 1) { - *pIndex = taosArrayGetSize(pWinInfos); - return addNewStateWindow(pWinInfos, ts, pKeyData, pCol); - } - *pIndex = index + 1; - return insertNewStateWindow(pWinInfos, ts, pKeyData, index + 1, pCol); + streamStateFreeCur(pCur); } -int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, uint64_t groupId, +int32_t updateStateWindowInfo(SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin, TSKEY* pTs, uint64_t groupId, SColumnInfoData* pKeyCol, int32_t rows, int32_t start, bool* allEqual, - SHashObj* pSeDeleted) { + SSHashObj* pResultRows, SSHashObj* pSeUpdated, SSHashObj* pSeDeleted) { *allEqual = true; - SStateWindowInfo* pWinInfo = taosArrayGet(pWinInfos, winIndex); for (int32_t i = start; i < rows; ++i) { char* pKeyData = colDataGetData(pKeyCol, i); if (!isTsInWindow(pWinInfo, pTs[i])) { if (isEqualStateKey(pWinInfo, pKeyData)) { - int32_t size = taosArrayGetSize(pWinInfos); - if (winIndex + 1 < size) { - SStateWindowInfo* pNextWin = taosArrayGet(pWinInfos, winIndex + 1); + if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) { // ts belongs to the next window - if (pTs[i] >= pNextWin->winInfo.win.skey) { + if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) { return i - start; } } @@ -4823,15 +4504,15 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u return i - start; } } - if (pWinInfo->winInfo.win.skey > pTs[i]) { + + if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { - SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId}; - taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); - pWinInfo->winInfo.isOutput = false; + saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); } - pWinInfo->winInfo.win.skey = pTs[i]; + removeSessionResult(pSeUpdated, pResultRows, pWinInfo->winInfo.sessionWin); + pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; } - pWinInfo->winInfo.win.ekey = TMAX(pWinInfo->winInfo.win.ekey, pTs[i]); + pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]); if (!isEqualStateKey(pWinInfo, pKeyData)) { *allEqual = false; } @@ -4839,36 +4520,13 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u return rows - start; } -static void doClearStateWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SHashObj* pSeUpdated, - SHashObj* pSeDeleted) { - SColumnInfoData* pTsColInfo = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pGroupColInfo = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - TSKEY* tsCol = (TSKEY*)pTsColInfo->pData; - bool allEqual = false; - int32_t step = 1; - uint64_t* gpCol = (uint64_t*)pGroupColInfo->pData; - for (int32_t i = 0; i < pBlock->info.rows; i += step) { - int32_t winIndex = 0; - SStateWindowInfo* pCurWin = getStateWindowByTs(pAggSup, tsCol[i], gpCol[i], &winIndex); - if (!pCurWin) { - continue; - } - updateSessionWindowInfo(&pCurWin->winInfo, tsCol, NULL, 0, pBlock->info.rows, i, 0, NULL); - taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); - } -} - -static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pSeUpdated, - SHashObj* pStDeleted) { +static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, + SSHashObj* pStDeleted) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamStateAggOperatorInfo* pInfo = pOperator->info; - bool masterScan = true; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int64_t groupId = pSDataBlock->info.groupId; int64_t code = TSDB_CODE_SUCCESS; - int32_t step = 1; - bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; @@ -4880,39 +4538,44 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - blockDataEnsureCapacity(pAggSup->pScanBlock, pSDataBlock->info.rows); + int32_t rows = pSDataBlock->info.rows; + blockDataEnsureCapacity(pAggSup->pScanBlock, rows); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); - for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { + for (int32_t i = 0; i < rows; i += winRows) { if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { i++; continue; } - char* pKeyData = colDataGetData(pKeyColInfo, i); - int32_t winIndex = 0; - bool allEqual = true; - SStateWindowInfo* pCurWin = getStateWindow(pAggSup, tsCols[i], groupId, pKeyData, &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows, - i, &allEqual, pStDeleted); + char* pKeyData = colDataGetData(pKeyColInfo, i); + int32_t winIndex = 0; + bool allEqual = true; + SStateWindowInfo curWin = {0}; + SStateWindowInfo nextWin = {0}; + setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin); + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); + winRows = updateStateWindowInfo(&curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual, + pAggSup->pResultRows, pSeUpdated, pStDeleted); if (!allEqual) { uint64_t uid = 0; - appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, - &uid, &groupId, NULL); - taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); - deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); + appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); + releaseOutputBuf(pAggSup->pState, NULL, (SResultRow*)curWin.winInfo.pOutputBuf); continue; } - code = doOneStateWindowAgg(pInfo, pSDataBlock, &pCurWin->winInfo, &pResult, i, winRows, numOfOutput, pOperator); + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, + pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->winInfo.isClosed = false; + saveSessionOutputBuf(pAggSup, &curWin.winInfo); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - SWinKey value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId}; - code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); + code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pCurWin->winInfo.isOutput = true; } } } @@ -4929,21 +4592,24 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single state"); + printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - if (pBInfo->pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - doSetOperatorCompleted(pOperator); + + doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "single state"); + return pBInfo->pRes; } - printDataBlock(pBInfo->pRes, "single state"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; + + doSetOperatorCompleted(pOperator); + return NULL; } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pSeUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + SSHashObj* pSeUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4951,18 +4617,16 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, "single state recv"); - if (pBlock->info.type == STREAM_CLEAR) { - doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted); - continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo); - copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { + SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); removeSessionResults(pSeUpdated, pWins); + copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForState); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated); continue; } @@ -4979,61 +4643,54 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState, - pInfo->ignoreExpiredData, destroyStateWinInfo); - // closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, destroyStateWinInfo); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated); copyUpdateResult(pSeUpdated, pUpdated); - taosHashCleanup(pSeUpdated); + removeSessionResults(pInfo->pSeDeleted, pUpdated); + tSimpleHashCleanup(pSeUpdated); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, "single state"); + printDataBlock(pInfo->pDelRes, "single state delete"); return pInfo->pDelRes; } - doBuildStreamResBlock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, "single state"); - return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; -} -int32_t initStateAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SStateWindowInfo)); + doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, "single state"); + return pBInfo->pRes; + } + doSetOperatorCompleted(pOperator); + return NULL; } SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStateKey)->pExpr; - int32_t code = TSDB_CODE_OUT_OF_MEMORY; + int32_t code = TSDB_CODE_SUCCESS; SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - SExprSupp* pSup = &pOperator->exprSupp; - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); - pInfo->stateCol = extractColumnFromColumnNode(pColNode); initResultSizeInfo(&pOperator->resultInfo, 4096); if (pStateNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); if (code != TSDB_CODE_SUCCESS) { goto _error; } } - initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType, @@ -5042,26 +4699,25 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + SExprSupp* pSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); code = initBasicInfoEx(&pInfo->binfo, pSup, pExprInfo, numOfCols, pResBlock); if (code != TSDB_CODE_SUCCESS) { goto _error; } - - code = initStateAggSupporter(&pInfo->streamAggSup, "StreamStateAggOperatorInfo", pSup->pCtx, numOfCols); + int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes; + int16_t type = pColNode->node.resType.type; + code = initStreamAggSupporter(&pInfo->streamAggSup, pSup->pCtx, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, + type); if (code != TSDB_CODE_SUCCESS) { goto _error; } - pInfo->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfCols, sizeof(SqlFunctionCtx)); - if (pInfo->pDummyCtx == NULL) { - goto _error; - } - - initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); pInfo->primaryTsIndex = tsSlotId; - pInfo->order = TSDB_ORDER_ASC; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); + pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pDelIterator = NULL; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; @@ -5078,7 +4734,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, + initDownStream(downstream, &pInfo->streamAggSup, pInfo->twAggSup.waterMark, pOperator->operatorType, pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5615,9 +5271,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge goto _error; } - int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); - SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc); + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, @@ -5646,6 +5301,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge goto _error; } + SSDataBlock* pResBlock = createResDataBlock(pIntervalPhyNode->window.node.pOutputDataBlockDesc); initBasicInfo(&pIntervalInfo->binfo, pResBlock); initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win); @@ -5677,7 +5333,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge return pOperator; _error: - destroyMergeIntervalOperatorInfo(pMergeIntervalInfo); + if (pMergeIntervalInfo != NULL) { + destroyMergeIntervalOperatorInfo(pMergeIntervalInfo); + } + taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -5829,18 +5488,21 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; pInfo->isFinal = false; - - pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; initResultSizeInfo(&pOperator->resultInfo, 4096); SExprSupp* pSup = &pOperator->exprSupp; - initBasicInfo(&pInfo->binfo, pResBlock); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 824e988c2f..b133041fdc 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -220,7 +220,7 @@ static int32_t doAddNewBucket(SLHashObj* pHashObj) { pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); - if (pBucket->pPageIdList == NULL || pBucket == NULL) { + if (pBucket->pPageIdList == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -256,6 +256,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, "", tsTempDir); if (code != 0) { + taosMemoryFree(pHashObj); terrno = code; return NULL; } @@ -351,7 +352,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void* key, size_t keyLen, void* data char* pStart = p->data; while (pStart - ((char*)p) < p->num) { SLHashNode* pNode = (SLHashNode*)pStart; - ASSERT(pNode->keyLen > 0 && pNode->dataLen >= 0); + ASSERT(pNode->keyLen > 0); char* k = GET_LHASH_NODE_KEY(pNode); int32_t hashv = pHashObj->hashFn(k, pNode->keyLen); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 16fd11f97d..b66a6f6770 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -16,9 +16,10 @@ #include "tsimplehash.h" #include "taoserror.h" #include "tlog.h" +#include "tdef.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16L) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) #define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) @@ -104,20 +105,20 @@ static void tSimpleHashTableResize(SSHashObj *pHashObj) { int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { - uDebug("current capacity:%zu, maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", + uDebug("current capacity:%"PRIzu", maximum capacity:%" PRIu64 ", no resize applied due to limitation is reached", pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, POINTER_BYTES * newCapacity); if (!pNewEntryList) { uWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc * sizeof(void *)); + memset((char *)pNewEntryList + pHashObj->capacity * POINTER_BYTES, 0, inc * sizeof(void *)); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 74da5c45ee..85582cbd39 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -548,7 +548,8 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock, false); code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId, pPageIdList); - if (code != 0) { + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pResList); return code; } } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index b5b7453e7a..dfdf03a5a8 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -118,6 +118,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pB int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); +EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow); EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow); int32_t lastRowFunction(SqlFunctionCtx* pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ce440a525a..2223954a5b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2441,6 +2441,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_FIRST, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLast, + .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, @@ -2455,6 +2456,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_FIRST_PARTIAL, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC, .translateFunc = translateFirstLastPartial, + .dynDataRequiredFunc = firstDynDataReq, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3255ef7d26..076cf13ad3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2506,6 +2506,8 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { int32_t start = pInput->startRowIndex; if (pInfo->algo == APERCT_ALGO_TDIGEST) { + buildTDigestInfo(pInfo); + tdigestAutoFill(pInfo->pTDigest, COMPRESSION); for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2519,12 +2521,11 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { tdigestAdd(pInfo->pTDigest, v, w); } } else { - qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, - numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); - // might be a race condition here that pHisto can be overwritten or setup function // has not been called, need to relink the buffer pHisto points to. buildHistogramInfo(pInfo); + qDebug("%s before add %d elements into histogram, total:%d, numOfEntry:%d, pHisto:%p, elems: %p", __FUNCTION__, + numOfElems, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, pInfo->pHisto, pInfo->pHisto->elems); for (int32_t i = start; i < pInput->numOfRows + start; ++i) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; @@ -2558,8 +2559,9 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* buildTDigestInfo(pOutput); TDigest* pTDigest = pOutput->pTDigest; + tdigestAutoFill(pTDigest, COMPRESSION); - if (pTDigest->num_centroids <= 0) { + if (pTDigest->num_centroids <= 0 && pTDigest->num_buffered_pts == 0) { memcpy(pTDigest, pInput->pTDigest, (size_t)TDIGEST_SIZE(COMPRESSION)); tdigestAutoFill(pTDigest, COMPRESSION); } else { @@ -2631,6 +2633,7 @@ int32_t apercentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (pInfo->algo == APERCT_ALGO_TDIGEST) { buildTDigestInfo(pInfo); + tdigestAutoFill(pInfo->pTDigest, COMPRESSION); if (pInfo->pTDigest->size > 0) { pInfo->result = tdigestQuantile(pInfo->pTDigest, pInfo->percent / 100); } else { // no need to free @@ -2696,6 +2699,22 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) return TSDB_CODE_SUCCESS; } +EFuncDataRequired firstDynDataReq(void* pRes, STimeWindow* pTimeWindow) { + SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; + + // not initialized yet, data is required + if (pEntry == NULL) { + return FUNC_DATA_REQUIRED_DATA_LOAD; + } + + SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry); + if (pResult->hasResult && pResult->ts <= pTimeWindow->skey) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } else { + return FUNC_DATA_REQUIRED_DATA_LOAD; + } +} + EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) { SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; @@ -5335,16 +5354,14 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t maxCount = 0; for (int32_t i = 0; i < pInfo->numOfPoints; ++i) { SModeItem* pItem = (SModeItem*)(pInfo->pItems + i * (sizeof(SModeItem) + pInfo->colBytes)); - if (pItem->count > maxCount) { + if (pItem->count >= maxCount) { maxCount = pItem->count; resIndex = i; - } else if (pItem->count == maxCount) { - resIndex = -1; } } SModeItem* pResItem = (SModeItem*)(pInfo->pItems + resIndex * (sizeof(SModeItem) + pInfo->colBytes)); - colDataAppend(pCol, currentRow, pResItem->data, (resIndex == -1) ? true : false); + colDataAppend(pCol, currentRow, pResItem->data, (maxCount == 0) ? true : false); return pResInfo->numOfRes; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 9605528ad6..4e7c0a0ef7 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -138,7 +138,7 @@ void idxReleaseRef(int64_t ref); #define IDX_TYPE_ADD_EXTERN_TYPE(ty, exTy) \ do { \ uint8_t oldTy = ty; \ - ty = (ty >> 4) | exTy; \ + ty = ((ty >> 4) & 0xFF) | exTy; \ ty = (ty << 4) | oldTy; \ } while (0) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index cb671cfff9..76dc84ae42 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -139,7 +139,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { END: if (idx != NULL) { - indexClose(idx); + indexDestroy(idx); } *index = NULL; return ret; diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 72f693f7e5..0bb454571a 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -538,7 +538,7 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { idxCacheRef(pCache); // encode data CacheTerm* ct = taosMemoryCalloc(1, sizeof(CacheTerm)); - if (cache == NULL) { + if (ct == NULL) { return -1; } // set up key @@ -730,15 +730,17 @@ static int32_t idxCacheJsonTermCompare(const void* l, const void* r) { return cmp; } static MemTable* idxInternalCacheCreate(int8_t type) { - int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : TSDB_DATA_TYPE_BINARY; + // int ttype = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? TSDB_DATA_TYPE_BINARY : + // TSDB_DATA_TYPE_BINARY; + int ttype = TSDB_DATA_TYPE_BINARY; int32_t (*cmpFn)(const void* l, const void* r) = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? idxCacheJsonTermCompare : idxCacheTermCompare; MemTable* tbl = taosMemoryCalloc(1, sizeof(MemTable)); idxMemRef(tbl); - if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) { - tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); - } + // if (ttype == TSDB_DATA_TYPE_BINARY || ttype == TSDB_DATA_TYPE_NCHAR) { + tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, ttype, MAX_INDEX_KEY_LEN, cmpFn, SL_ALLOW_DUP_KEY, idxCacheTermGet); + //} return tbl; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index ecf9136073..25036996fc 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -427,6 +427,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint64_t*)src, *dst, 1); tlen = strlen(*dst); + break; case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6d81499dbc..075408f1b3 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -231,7 +231,9 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; param->colValType = (uint8_t)(vn->node.resType.type); - memcpy(param->colName, vn->literal, strlen(vn->literal)); + if (strlen(vn->literal) <= sizeof(param->colName)) { + memcpy(param->colName, vn->literal, strlen(vn->literal)); + } break; } case QUERY_NODE_COLUMN: { @@ -400,54 +402,52 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int8_t ltype = left->colValType, rtype = right->colValType; if (ltype == TSDB_DATA_TYPE_FLOAT) { - float f; + float f = 0; SIF_DATA_CONVERT(rtype, right->condValue, f); typedata->f = f; param->val = &typedata->f; } else if (ltype == TSDB_DATA_TYPE_DOUBLE) { - double d; + double d = 0; SIF_DATA_CONVERT(rtype, right->condValue, d); typedata->d = d; param->val = &typedata->d; } else if (ltype == TSDB_DATA_TYPE_BIGINT) { - int64_t i64; + int64_t i64 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i64); typedata->i64 = i64; param->val = &typedata->i64; } else if (ltype == TSDB_DATA_TYPE_INT) { - int32_t i32; + int32_t i32 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i32); typedata->i32 = i32; param->val = &typedata->i32; } else if (ltype == TSDB_DATA_TYPE_SMALLINT) { - int16_t i16; - + int16_t i16 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i16); typedata->i16 = i16; param->val = &typedata->i16; } else if (ltype == TSDB_DATA_TYPE_TINYINT) { - int8_t i8; + int8_t i8 = 0; SIF_DATA_CONVERT(rtype, right->condValue, i8) typedata->i8 = i8; param->val = &typedata->i8; } else if (ltype == TSDB_DATA_TYPE_UBIGINT) { - uint64_t u64; + uint64_t u64 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u64); typedata->u64 = u64; param->val = &typedata->u64; - } else if (ltype == TSDB_DATA_TYPE_UINT) { - uint32_t u32; + uint32_t u32 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u32); typedata->u32 = u32; param->val = &typedata->u32; } else if (ltype == TSDB_DATA_TYPE_USMALLINT) { - uint16_t u16; + uint16_t u16 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u16); typedata->u16 = u16; param->val = &typedata->u16; } else if (ltype == TSDB_DATA_TYPE_UTINYINT) { - uint8_t u8; + uint8_t u8 = 0; SIF_DATA_CONVERT(rtype, right->condValue, u8); typedata->u8 = u8; param->val = &typedata->u8; @@ -663,7 +663,7 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { // ugly code, refactor later if (nParam > 1 && params[1].status == SFLT_NOT_INDEX) { output->status = SFLT_NOT_INDEX; - return code; + goto _return; } SIF_ERR_JRET(sifGetOperFn(node->opType, &operFn, &output->status)); } diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 01dffa782d..aed2ec3ee3 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -338,7 +338,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) { return v; } // 0 indicate that common_input is None - return v == 0 ? 0 : COMMON_INPUT(v); + return COMMON_INPUT(v); } // input_len diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 7021fdfae3..33960ad8c2 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -72,7 +72,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of if (offset >= ctx->file.size) return 0; do { - char key[128] = {0}; + char key[1024] = {0}; + assert(strlen(ctx->file.buf) + 1 + 64 < sizeof(key)); idxGenLRUKey(key, ctx->file.buf, blkId); LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key)); @@ -99,6 +100,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of assert(blk->nread <= kBlockSize); if (blk->nread < kBlockSize && blk->nread < len) { + taosMemoryFree(blk); break; } @@ -150,7 +152,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int if (ctx->type == TFILE) { // ugly code, refactor later ctx->file.readOnly = readOnly; - memcpy(ctx->file.buf, path, strlen(path)); + memcpy(ctx->file.buf, path, sizeof(ctx->file.buf)); if (readOnly == false) { ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); taosFtruncateFile(ctx->file.pFile, 0); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e746bb4303..4fdf6d9e57 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -506,7 +506,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c tfh.suid = suid; tfh.version = version; tfh.colType = colType; - memcpy(tfh.colName, colName, strlen(colName)); + if (strlen(colName) <= sizeof(tfh.colName)) { + memcpy(tfh.colName, colName, strlen(colName)); + } return tfileWriterCreate(wcx, &tfh); } @@ -580,8 +582,14 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) { if (cap < ttsz) { cap = ttsz; - buf = (char*)taosMemoryRealloc(buf, cap); + char* t = (char*)taosMemoryRealloc(buf, cap); + if (t == NULL) { + taosMemoryFree(buf); + return -1; + } + buf = t; } + char* p = buf; tfileSerialTableIdsToBuf(p, v->tableId); tw->ctx->write(tw->ctx, buf, ttsz); diff --git a/source/libs/monitor/src/monMain.c b/source/libs/monitor/src/monMain.c index eab5a33450..890a59e4be 100644 --- a/source/libs/monitor/src/monMain.c +++ b/source/libs/monitor/src/monMain.c @@ -401,7 +401,6 @@ static void monGenDnodeJson(SMonInfo *pMonitor) { tjsonAddDoubleToObject(pJson, "has_mnode", pInfo->has_mnode); tjsonAddDoubleToObject(pJson, "has_qnode", pInfo->has_qnode); tjsonAddDoubleToObject(pJson, "has_snode", pInfo->has_snode); - tjsonAddDoubleToObject(pJson, "has_bnode", pInfo->has_bnode); } static void monGenDiskJson(SMonInfo *pMonitor) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5b861ef79a..f59a6c7f9b 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -543,7 +543,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(uv)) { + } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } uint8_t tmpVal = (uint8_t)uv; @@ -563,7 +563,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_USMALLINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(uv)) { + } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } uint16_t tmpVal = (uint16_t)uv; @@ -583,7 +583,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(uv)) { + } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } uint32_t tmpVal = (uint32_t)uv; @@ -600,8 +600,6 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT(uv)) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } return func(pMsgBuf, &uv, pSchema->bytes, param); } @@ -844,7 +842,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UTINYINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); - } else if (!IS_VALID_UTINYINT(uv)) { + } else if (uv > UINT8_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); } *(uint8_t*)(&val->i64) = uv; @@ -864,7 +862,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_USMALLINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); - } else if (!IS_VALID_USMALLINT(uv)) { + } else if (uv > UINT16_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); } *(uint16_t*)(&val->i64) = uv; @@ -884,7 +882,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); - } else if (!IS_VALID_UINT(uv)) { + } else if (uv > UINT32_MAX) { return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); } *(uint32_t*)(&val->i64) = uv; @@ -902,8 +900,6 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, int16 case TSDB_DATA_TYPE_UBIGINT: { if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); - } else if (!IS_VALID_UBIGINT(uv)) { - return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); } *(uint64_t*)(&val->i64) = uv; break; @@ -994,7 +990,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint isParseBindParam = true; if (NULL == pCxt->pStmtCb) { code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); - goto end; + break; } continue; @@ -1002,57 +998,50 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint if (isParseBindParam) { code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); - goto end; + break; } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // todo this can be optimize with parse column code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - if (!isNullValue(pTagSchema->type, &sToken)) { - taosArrayPush(tagName, pTagSchema->name); - } - if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { - if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); - goto end; + if (TSDB_CODE_SUCCESS == code) { + if (!isNullValue(pTagSchema->type, &sToken)) { + taosArrayPush(tagName, pTagSchema->name); } - if (isNullValue(pTagSchema->type, &sToken)) { - code = tTagNew(pTagVals, 1, true, &pTag); + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + isJson = true; + if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); + break; + } + if (isNullValue(pTagSchema->type, &sToken)) { + code = tTagNew(pTagVals, 1, true, &pTag); + } else { + code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + } } else { - code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); + STagVal val = {0}; + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pTagVals, &val); + } } - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - isJson = true; - } else { - STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); - if (TSDB_CODE_SUCCESS != code) { - goto end; - } - - taosArrayPush(pTagVals, &val); + } + if (TSDB_CODE_SUCCESS != code) { + break; } } - if (isParseBindParam) { - code = TSDB_CODE_SUCCESS; - goto end; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { + code = tTagNew(pTagVals, 1, false, &pTag); } - if (!isJson && (code = tTagNew(pTagVals, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { - goto end; + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { + buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, + pCxt->pTableMeta->tableInfo.numOfTags); + pTag = NULL; } - buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid, pCxt->sTableName, tagName, - pCxt->pTableMeta->tableInfo.numOfTags); - -end: for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal* p = (STagVal*)taosArrayGet(pTagVals, i); if (IS_VAR_DATA_TYPE(p->type)) { @@ -1061,6 +1050,7 @@ end: } taosArrayDestroy(pTagVals); taosArrayDestroy(tagName); + tTagFree(pTag); return code; } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 997e680c99..09e40cad60 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -244,7 +244,7 @@ int32_t getDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32 } } - taosHashPut(pHashList, (const char*)id, idLen, (char*)dataBlocks, POINTER_BYTES); + taosHashPut(pHashList, id, idLen, dataBlocks, POINTER_BYTES); if (pBlockList) { taosArrayPush(pBlockList, dataBlocks); } @@ -666,7 +666,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } } else { for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; + void* payload = (blkKeyTuple + i)->payloadAddr; TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); memcpy(pDataBlock, payload, rowTLen); pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 76c01cfd90..afcd3610f4 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -597,6 +597,8 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { *tokenId = TK_NK_BOOL; return i; } + *tokenId = tKeywordCode(z, i); + return i; } default: { if (((*z & 0x80) != 0) || !isIdChar[(uint8_t)*z]) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index dbe0c97ab6..5a05440c73 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -995,13 +995,17 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } + int64_t value = 0; if (IS_UNSIGNED_NUMERIC_TYPE(pVal->node.resType.type)) { - pVal->datum.i = (int64_t)pVal->datum.u; + value = pVal->datum.u; } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { - pVal->datum.i = (int64_t)pVal->datum.d; + value = pVal->datum.d; } else if (TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { - pVal->datum.i = pVal->datum.b; + value = pVal->datum.b; + } else { + value = pVal->datum.i; } + pVal->datum.i = value; return TSDB_CODE_SUCCESS; } else if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_TIMESTAMP == pVal->node.resType.type) { if (TSDB_CODE_SUCCESS == taosParseTime(pVal->literal, &pVal->datum.i, pVal->node.resType.bytes, @@ -1083,7 +1087,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UTINYINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UTINYINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT8_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint8_t*)&pVal->typeData = pVal->datum.u; @@ -1091,7 +1095,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_USMALLINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_USMALLINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT16_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint16_t*)&pVal->typeData = pVal->datum.u; @@ -1099,7 +1103,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UINT(pVal->datum.u))) { + if (strict && (TSDB_CODE_SUCCESS != code || pVal->datum.u > UINT32_MAX)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint32_t*)&pVal->typeData = pVal->datum.u; @@ -1107,7 +1111,7 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_UBIGINT: { code = toUInteger(pVal->literal, strlen(pVal->literal), 10, &pVal->datum.u); - if (strict && (TSDB_CODE_SUCCESS != code || !IS_VALID_UBIGINT(pVal->datum.u))) { + if (strict && TSDB_CODE_SUCCESS != code) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } *(uint64_t*)&pVal->typeData = pVal->datum.u; @@ -2357,7 +2361,8 @@ static int32_t setTableIndex(STranslateContext* pCxt, SName* pName, SRealTableNo } static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + if ((!pSelect->hasLastRowFunc && !pSelect->hasLastFunc) || QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || + TSDB_SYSTEM_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType) { return TSDB_CODE_SUCCESS; } @@ -4244,7 +4249,7 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN code = taosHashPut(pHash, pTag->colName, len, &pTag, POINTER_BYTES); } if (TSDB_CODE_SUCCESS == code) { - tagsSize += pTag->dataType.bytes; + tagsSize += calcTypeBytes(pTag->dataType); } else { break; } @@ -4295,7 +4300,7 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES); } if (TSDB_CODE_SUCCESS == code) { - rowSize += pCol->dataType.bytes; + rowSize += calcTypeBytes(pCol->dataType); } else { break; } @@ -5161,7 +5166,7 @@ static int16_t getCreateComponentNodeMsgType(ENodeType type) { static int32_t translateCreateComponentNode(STranslateContext* pCxt, SCreateComponentNodeStmt* pStmt) { SMCreateQnodeReq createReq = {.dnodeId = pStmt->dnodeId}; return buildCmdMsg(pCxt, getCreateComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &createReq); + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &createReq); } static int16_t getDropComponentNodeMsgType(ENodeType type) { @@ -5183,7 +5188,7 @@ static int16_t getDropComponentNodeMsgType(ENodeType type) { static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponentNodeStmt* pStmt) { SDDropQnodeReq dropReq = {.dnodeId = pStmt->dnodeId}; return buildCmdMsg(pCxt, getDropComponentNodeMsgType(nodeType(pStmt)), - (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq); + (FSerializeFunc)tSerializeSCreateDropMQSNodeReq, &dropReq); } static int32_t checkTopicQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index ee82e6c04b..95eb094edb 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1200,7 +1200,7 @@ int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* p int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo); SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex); if (TSDB_CODE_SUCCESS == pRes->code) { - *pMeta = tableMetaDup(pRes->pRes); + *pMeta = tableMetaDup((const STableMeta*)pRes->pRes); if (NULL == *pMeta) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 121bbaa733..3ea6ae3db4 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -27,25 +27,6 @@ TEST_F(ParserInitialCTest, createAccount) { run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } -TEST_F(ParserInitialCTest, createBnode) { - useDb("root", "test"); - - SMCreateQnodeReq expect = {0}; - - auto setCreateQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; - - setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { - ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_BNODE_STMT); - SMCreateQnodeReq req = {0}; - ASSERT_TRUE(TSDB_CODE_SUCCESS == - tDeserializeSCreateDropMQSBNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); - ASSERT_EQ(req.dnodeId, expect.dnodeId); - }); - - setCreateQnodeReq(1); - run("CREATE BNODE ON DNODE 1"); -} - /* * CREATE DATABASE [IF NOT EXISTS] db_name [database_options] * diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 870afa694a..c7865f3da3 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -52,12 +52,6 @@ TEST_F(ParserInitialDTest, describe) { // todo describe // todo DROP account -TEST_F(ParserInitialDTest, dropBnode) { - useDb("root", "test"); - - run("DROP BNODE ON DNODE 1"); -} - // DROP CONSUMER GROUP [ IF EXISTS ] cgroup_name ON topic_name TEST_F(ParserInitialDTest, dropConsumerGroup) { useDb("root", "test"); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7a145e7dde..f5d32796b2 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -124,9 +124,8 @@ static void optSetParentOrder(SLogicNode* pNode, EOrder order) { EDealRes scanPathOptHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - // *((bool*)pContext) = - // (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); - *((bool*)pContext) = true; + *((bool*)pContext) = + (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -1521,11 +1520,15 @@ static bool partTagsHasIndefRowsSelectFunc(SNodeList* pFuncs) { return false; } -static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, SNodeList* pAggFuncs) { +static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t start, SNodeList* pAggFuncs) { bool hasIndefRowsSelectFunc = partTagsHasIndefRowsSelectFunc(pAggFuncs); int32_t code = TSDB_CODE_SUCCESS; + int32_t index = 0; SNode* pNode = NULL; FOREACH(pNode, pGroupTags) { + if (index++ < start) { + continue; + } if (hasIndefRowsSelectFunc) { code = nodesListStrictAppend(pAggFuncs, partTagsCreateWrapperFunc("_select_value", pNode)); } else { @@ -1560,20 +1563,35 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } else { SAggLogicNode* pAgg = (SAggLogicNode*)pNode; + int32_t start = -1; SNode* pGroupKey = NULL; FOREACH(pGroupKey, pAgg->pGroupKeys) { - code = nodesListMakeStrictAppend( - &pScan->pGroupTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + SNode* pGroupExpr = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + if (NULL != pScan->pGroupTags) { + SNode* pGroupTag = NULL; + FOREACH(pGroupTag, pScan->pGroupTags) { + if (nodesEqualNode(pGroupTag, pGroupExpr)) { + continue; + } + } + } + if (start < 0) { + start = LIST_LENGTH(pScan->pGroupTags); + } + code = nodesListMakeStrictAppend(&pScan->pGroupTags, nodesCloneNode(pGroupExpr)); if (TSDB_CODE_SUCCESS != code) { break; } } NODES_DESTORY_LIST(pAgg->pGroupKeys); - code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, pAgg->pAggFuncs); + if (TSDB_CODE_SUCCESS == code && start >= 0) { + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg->pAggFuncs); + } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pGroupTags); } + pCxt->optimized = true; return code; } @@ -1960,7 +1978,8 @@ static SNode* rewriteUniqueOptCreateFirstFunc(SFunctionNode* pSelectValue, SNode if (NULL != pSelectValue) { strcpy(pFunc->node.aliasName, pSelectValue->node.aliasName); } else { - snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%p", pFunc->functionName, (void*)pFunc); + int64_t pointer = (int64_t)pFunc; + snprintf(pFunc->node.aliasName, sizeof(pFunc->node.aliasName), "%s.%" PRId64 "", pFunc->functionName, pointer); } int32_t code = nodesListMakeStrictAppend(&pFunc->pParameterList, nodesCloneNode(pCol)); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 9567a2388f..a2bbc0af60 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -342,7 +342,8 @@ static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWStart->functionName, "_wstart"); - snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, (void*)pWStart); + int64_t pointer = (int64_t)pWStart; + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%" PRId64 "", pWStart->functionName, pointer); int32_t code = fmGetFuncInfo(pWStart, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pFuncs, (SNode*)pWStart); @@ -367,7 +368,8 @@ static int32_t stbSplAppendWEnd(SWindowLogicNode* pWin, int32_t* pIndex) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pWEnd->functionName, "_wend"); - snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%p", pWEnd->functionName, (void*)pWEnd); + int64_t pointer = (int64_t)pWEnd; + snprintf(pWEnd->node.aliasName, sizeof(pWEnd->node.aliasName), "%s.%" PRId64 "", pWEnd->functionName, pointer); int32_t code = fmGetFuncInfo(pWEnd, NULL, 0); if (TSDB_CODE_SUCCESS == code) { code = nodesListStrictAppend(pWin->pFuncs, (SNode*)pWEnd); diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 48ab1e1ac2..3d0467dffe 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -61,6 +61,8 @@ TEST_F(PlanPartitionByTest, withGroupBy) { run("SELECT COUNT(*) FROM t1 PARTITION BY c1 GROUP BY c2"); run("SELECT TBNAME, c1 FROM st1 PARTITION BY TBNAME GROUP BY c1"); + + run("SELECT COUNT(*) FROM t1 PARTITION BY TBNAME GROUP BY TBNAME"); } TEST_F(PlanPartitionByTest, withTimeLineFunc) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 8162b922ce..618f597d72 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -357,8 +357,7 @@ char* parseTagDatatoJson(void* p) { for (int j = 0; j < nCols; ++j) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); // json key encode by binary - memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); + tstrncpy(tagJsonKey, pTagVal->pKey, sizeof(tagJsonKey)); // json value char type = pTagVal->type; if (type == TSDB_DATA_TYPE_NULL) { @@ -424,6 +423,12 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } + if ((pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) > TSDB_MAX_COL_TAG_NUM) { + *pDst = NULL; + qError("too many column and tag num:%d,%d", pSrc->tableInfo.numOfColumns, pSrc->tableInfo.numOfTags); + return TSDB_CODE_INVALID_PARA; + } + int32_t metaSize = sizeof(STableMeta) + (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 80ba356e8f..fadd07a9f3 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -173,7 +173,7 @@ int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t } SDbCfgReq dbCfgReq = {0}; - strcpy(dbCfgReq.db, input); + strncpy(dbCfgReq.db, input, sizeof(dbCfgReq.db) - 1); int32_t bufLen = tSerializeSDbCfgReq(NULL, 0, &dbCfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -191,7 +191,7 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t } SUserIndexReq indexReq = {0}; - strcpy(indexReq.indexFName, input); + strncpy(indexReq.indexFName, input, sizeof(indexReq.indexFName) - 1); int32_t bufLen = tSerializeSUserIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -233,7 +233,7 @@ int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32 } SGetUserAuthReq req = {0}; - strncpy(req.user, input, sizeof(req.user)); + strncpy(req.user, input, sizeof(req.user) - 1); int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); void *pBuf = (*mallcFp)(bufLen); @@ -251,7 +251,7 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ } STableIndexReq indexReq = {0}; - strcpy(indexReq.tbFName, input); + strncpy(indexReq.tbFName, input, sizeof(indexReq.tbFName) - 1); int32_t bufLen = tSerializeSTableIndexReq(NULL, 0, &indexReq); void *pBuf = (*mallcFp)(bufLen); @@ -271,8 +271,8 @@ int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t SBuildTableInput *pInput = input; STableCfgReq cfgReq = {0}; cfgReq.header.vgId = pInput->vgId; - strcpy(cfgReq.dbFName, pInput->dbFName); - strcpy(cfgReq.tbName, pInput->tbName); + strncpy(cfgReq.dbFName, pInput->dbFName, sizeof(cfgReq.dbFName) - 1); + strncpy(cfgReq.tbName, pInput->tbName, sizeof(cfgReq.tbName) - 1); int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); void *pBuf = (*mallcFp)(bufLen); @@ -615,6 +615,8 @@ int32_t queryProcessGetTbCfgRsp(void *output, char *msg, int32_t msgSize) { STableCfgRsp *out = taosMemoryCalloc(1, sizeof(STableCfgRsp)); if (tDeserializeSTableCfgRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableCfgRsp failed, msgSize:%d", msgSize); + tFreeSTableCfgRsp(out); + taosMemoryFree(out); return TSDB_CODE_INVALID_MSG; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 3038b87930..e9ded9b269 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -412,7 +412,7 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { while (true) { paramIdx = atomic_load_32(&gQwMgmt.paramIdx); if (paramIdx == tListLen(gQwMgmt.param)) { - newParamIdx = 0; + newParamIdx = 1; } else { newParamIdx = paramIdx + 1; } @@ -422,6 +422,10 @@ void qwSetHbParam(int64_t refId, SQWHbParam **pParam) { } } + if (paramIdx == tListLen(gQwMgmt.param)) { + paramIdx = 0; + } + gQwMgmt.param[paramIdx].qwrId = gQwMgmt.qwRef; gQwMgmt.param[paramIdx].refId = refId; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index d2c16a1cb5..3df16563e2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -258,7 +258,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); if (len < 0) { - QW_TASK_ELOG("invalid length from dsGetDataLength, length:%d", len); + QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -292,7 +292,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } // Got data from sink - QW_TASK_DLOG("there are data in sink, dataLength:%d", len); + QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64, len); *dataLen += len; @@ -398,7 +398,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_ELOG("task already dropped at wrong phase %s", qwPhaseStr(phase)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_STATUS_ERROR); - break; } if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { @@ -408,7 +407,6 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu // QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); - break; } QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC)); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 81ee173648..a04d5a3a3a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1214,7 +1214,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u, uint32_t *uidx) { SFilterFieldId left, right, *pright = &right; - int32_t type = FILTER_UNIT_DATA_TYPE(u); + uint8_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = 0; filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); @@ -1644,7 +1644,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); SColumnNode *refNode = (SColumnNode *)left->desc; - if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr <= OP_TYPE_JSON_CONTAINS) { len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr)); } @@ -1664,7 +1664,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); - if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { + if (unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS) { sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, operatorTypeStr(unit->compare.optr2)); } @@ -1709,9 +1709,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) ctx->isrange); if (ctx->isrange) { SFilterRangeNode *r = ctx->rs; + int32_t tlen = 0; while (r) { - char str[256] = {0}; - int32_t tlen = 0; + char str[256] = {0}; if (FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { strcat(str, "(NULL)"); } else { @@ -2614,7 +2614,7 @@ int32_t filterRewrite(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t gResNum int32_t usize = (int32_t)taosArrayGetSize((SArray *)colInfo->info); for (int32_t n = 0; n < usize; ++n) { - SFilterUnit *u = taosArrayGetP((SArray *)colInfo->info, n); + SFilterUnit *u = (SFilterUnit *)taosArrayGetP((SArray *)colInfo->info, n); filterAddUnitFromUnit(info, &oinfo, u, &uidx); filterAddUnitToGroup(&ng, uidx); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 1f09fd4799..cc1949f17d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -629,6 +629,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } @@ -664,6 +666,8 @@ int32_t sclWalkWhenList(SScalarCtx *ctx, SNodeList* pList, struct SListCell* pCe sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFreeClear(pWhen); + taosMemoryFreeClear(pThen); } if (pElse) { @@ -688,6 +692,8 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } @@ -929,6 +935,10 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp sclFreeParam(&comp); sclFreeParam(pWhen); sclFreeParam(pThen); + taosMemoryFree(pCase); + taosMemoryFree(pElse); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); return TSDB_CODE_SUCCESS; @@ -940,6 +950,10 @@ _return: sclFreeParam(pWhen); sclFreeParam(pThen); sclFreeParam(output); + taosMemoryFree(pCase); + taosMemoryFree(pElse); + taosMemoryFree(pWhen); + taosMemoryFree(pThen); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f467721248..f089bad04e 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -654,7 +654,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, return TSDB_CODE_APP_ERROR; } - int32_t rstart = startIndex >= 0 ? startIndex : 0; + int32_t rstart = (startIndex >= 0 && startIndex < pIn->numOfRows) ? startIndex : 0; int32_t rend = numOfRows > 0 ? rstart + numOfRows - 1 : rstart + pIn->numOfRows - 1; SSclVectorConvCtx cCtx = {pIn, pOut, rstart, rend, pInputCol->info.type, pOutputCol->info.type}; @@ -862,25 +862,25 @@ int32_t vectorConvertSingleColImpl(const SScalarParam* pIn, SScalarParam* pOut, int8_t gConvertTypes[TSDB_DATA_TYPE_BLOB + 1][TSDB_DATA_TYPE_BLOB + 1] = { /* NULL BOOL TINY SMAL INT BIG FLOA DOUB VARC TIME NCHA UTIN USMA UINT UBIG JSON VARB DECI BLOB */ - /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 7, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, - /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 7, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, - /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 7, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, - /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 7, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, - /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, - /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, - /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, - /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, - /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, - /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, - /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, - /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, - /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, - /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + /*NULL*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BOOL*/ 0, 0, 2, 3, 4, 5, 6, 7, 5, 9, 7, 11, 12, 13, 14, 0, 7, 0, 0, + /*TINY*/ 0, 0, 0, 3, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, + /*SMAL*/ 0, 0, 0, 0, 4, 5, 6, 7, 5, 9, 7, 3, 4, 5, 7, 0, 7, 0, 0, + /*INT */ 0, 0, 0, 0, 0, 5, 6, 7, 5, 9, 7, 4, 4, 5, 7, 0, 7, 0, 0, + /*BIGI*/ 0, 0, 0, 0, 0, 0, 6, 7, 5, 9, 7, 5, 5, 5, 7, 0, 7, 0, 0, + /*FLOA*/ 0, 0, 0, 0, 0, 0, 0, 7, 7, 6, 7, 6, 6, 6, 6, 0, 7, 0, 0, + /*DOUB*/ 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7, 0, 7, 0, 0, + /*VARC*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8, 7, 7, 7, 7, 0, 0, 0, 0, + /*TIME*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 7, 0, 7, 0, 0, + /*NCHA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 0, 0, 0, + /*UTIN*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 13, 14, 0, 7, 0, 0, + /*USMA*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 14, 0, 7, 0, 0, + /*UINT*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 7, 0, 0, + /*UBIG*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, + /*JSON*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*VARB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*DECI*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /*BLOB*/ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; int32_t vectorGetConvertType(int32_t type1, int32_t type2) { if (type1 == type2) { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index bb8451ba37..8fcfdce2b2 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -122,7 +122,6 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_DROP: SCH_ERR_JRET(TSDB_CODE_QRY_JOB_FREED); - break; default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1b59c06140..c26ae4e646 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -169,7 +169,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) { SCH_LOCK(SCH_WRITE, &pJob->resLock); if (NULL == pJob->execRes.res) { - pJob->execRes.res = taosArrayInit(batchRsp.nRsps, POINTER_BYTES); + pJob->execRes.res = (void*)taosArrayInit(batchRsp.nRsps, POINTER_BYTES); pJob->execRes.msgType = TDMT_VND_CREATE_TABLE; } @@ -386,7 +386,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa // NEVER REACH HERE SCH_TASK_ELOG("invalid status to handle drop task rsp, refId:0x%" PRIx64, pJob->refId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } case TDMT_SCH_LINK_BROKEN: SCH_TASK_ELOG("link broken received, error:%x - %s", rspCode, tstrerror(rspCode)); @@ -981,7 +980,7 @@ _return: } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { - uint32_t msgSize = 0; + int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; @@ -1133,12 +1132,11 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, default: SCH_TASK_ELOG("unknown msg type to send, msgType:%d", msgType); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - break; } #if 1 SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; - code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); + code = schAsyncSendMsg(pJob, pTask, &trans, addr, msgType, msg, (uint32_t)msgSize, persistHandle, (rpcCtx.args ? &rpcCtx : NULL)); msg = NULL; SCH_ERR_JRET(code); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 33bf50d853..b585373b0a 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -430,7 +430,8 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 if (SCH_IS_DATA_BIND_TASK(pTask)) { if (NULL == pData->pEpSet) { SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); - SCH_ERR_JRET(rspCode); + code = rspCode; + goto _return; } } @@ -904,7 +905,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { } else if (tsQueryPlannerTrace) { char *msg = NULL; int32_t msgLen = 0; - qSubPlanToString(plan, &msg, &msgLen); + SCH_ERR_RET(qSubPlanToString(plan, &msg, &msgLen)); SCH_TASK_DLOGL("physical plan len:%d, %s", msgLen, msg); taosMemoryFree(msg); } @@ -926,6 +927,7 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { } SArray *explainRes = NULL; + int32_t code = 0; SQWMsg qwMsg = {0}; qwMsg.msgInfo.taskType = TASK_TYPE_TEMP; qwMsg.msgInfo.explain = SCH_IS_EXPLAIN_JOB(pJob); @@ -938,14 +940,21 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalQuery(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &qwMsg, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } - SCH_RET(schProcessOnTaskSuccess(pJob, pTask)); + SCH_ERR_RET(schProcessOnTaskSuccess(pJob, pTask)); + +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } int32_t schLaunchTaskImpl(void *param) { @@ -1097,22 +1106,28 @@ int32_t schExecRemoteFetch(SSchJob *pJob, SSchTask *pTask) { int32_t schExecLocalFetch(SSchJob *pJob, SSchTask *pTask) { void *pRsp = NULL; + int32_t code = 0; SArray *explainRes = NULL; if (SCH_IS_EXPLAIN_JOB(pJob)) { explainRes = taosArrayInit(pJob->taskNum, sizeof(SExplainLocalRsp)); } - SCH_ERR_RET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, + SCH_ERR_JRET(qWorkerProcessLocalFetch(schMgmt.queryMgmt, schMgmt.sId, pJob->queryId, pTask->taskId, pJob->refId, pTask->execId, &pRsp, explainRes)); if (SCH_IS_EXPLAIN_JOB(pJob)) { SCH_ERR_RET(schHandleExplainRes(explainRes)); + explainRes = NULL; } SCH_ERR_RET(schProcessFetchRsp(pJob, pTask, pRsp, TSDB_CODE_SUCCESS)); - return TSDB_CODE_SUCCESS; +_return: + + taosArrayDestroy(explainRes); + + SCH_RET(code); } // Note: no more error processing, handled in function internal diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 7099dbb2c0..e20ed9d58e 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -242,7 +242,7 @@ uint64_t schGenUUID(void) { if (hashId == 0) { char uid[64] = {0}; - int32_t code = taosGetSystemUUID(uid, tListLen(uid)); + int32_t code = taosGetSystemUUID(uid, tListLen(uid) - 1); if (code != TSDB_CODE_SUCCESS) { qError("Failed to get the system uid, reason:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } else { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index c0ed135d74..6991f00bb7 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -24,6 +24,40 @@ typedef struct SStateKey { int64_t opNum; } SStateKey; +typedef struct SStateSessionKey { + SSessionKey key; + int64_t opNum; +} SStateSessionKey; + +static inline int sessionKeyCmpr(const SSessionKey* pWin1, const SSessionKey* pWin2) { + if (pWin1->groupId > pWin2->groupId) { + return 1; + } else if (pWin1->groupId < pWin2->groupId) { + return -1; + } + + if (pWin1->win.skey > pWin2->win.ekey) { + return 1; + } else if (pWin1->win.ekey < pWin2->win.skey) { + return -1; + } + + return 0; +} + +static inline int stateSessionKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { + SStateSessionKey* pWin1 = (SStateSessionKey*)pKey1; + SStateSessionKey* pWin2 = (SStateSessionKey*)pKey2; + + if (pWin1->opNum > pWin2->opNum) { + return 1; + } else if (pWin1->opNum < pWin2->opNum) { + return -1; + } + + return sessionKeyCmpr(&pWin1->key, &pWin2->key); +} + static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { SStateKey* pWin1 = (SStateKey*)pKey1; SStateKey* pWin2 = (SStateKey*)pKey2; @@ -79,6 +113,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int goto _err; } + if (tdbTbOpen("session.state.db", sizeof(SStateSessionKey), -1, stateSessionKeyCmpr, pState->db, + &pState->pSessionStateDb) < 0) { + goto _err; + } + if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) { goto _err; } @@ -95,6 +134,7 @@ _err: tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); + tdbTbClose(pState->pSessionStateDb); tdbClose(pState->db); taosMemoryFree(pState); return NULL; @@ -105,6 +145,7 @@ void streamStateClose(SStreamState* pState) { tdbTbClose(pState->pStateDb); tdbTbClose(pState->pFuncStateDb); tdbTbClose(pState->pFillStateDb); + tdbTbClose(pState->pSessionStateDb); tdbClose(pState->db); taosMemoryFree(pState); @@ -241,11 +282,11 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { if (pCur == NULL) return NULL; tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c = 0; SStateKey sKey = {.key = *key, .opNum = pState->number}; tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c); if (c != 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } pCur->number = pState->number; @@ -257,7 +298,7 @@ SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) if (pCur == NULL) return NULL; tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c = 0; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); if (c != 0) { streamStateFreeCur(pCur); @@ -348,21 +389,21 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key } pCur->number = pState->number; if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } SStateKey sKey = {.key = *key, .opNum = pState->number}; - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c > 0) return pCur; if (tdbTbcMoveToNext(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -375,20 +416,20 @@ SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* return NULL; } if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c > 0) return pCur; if (tdbTbcMoveToNext(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -401,20 +442,20 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* return NULL; } if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } - int32_t c; + int32_t c = 0; if (tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c) < 0) { tdbTbcClose(pCur->pCur); - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } if (c < 0) return pCur; if (tdbTbcMoveToPrev(pCur->pCur) < 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } @@ -445,3 +486,229 @@ void streamStateFreeCur(SStreamStateCur* pCur) { } void streamFreeVal(void* val) { tdbFree(val); } + +int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbUpsert(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, &pState->txn); +} + +SStreamStateCur* streamStateSessionGetRanomCur(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) return NULL; + tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL); + + int32_t c = 0; + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c); + if (c != 0) { + streamStateFreeCur(pCur); + return NULL; + } + pCur->number = pState->number; + return pCur; +} + +int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + void* tmp = NULL; + if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + *pVal = tdbRealloc(NULL, *pVLen); + memcpy(*pVal, tmp, *pVLen); + streamStateFreeCur(pCur); + return 0; + } + streamStateFreeCur(pCur); + return -1; +} + +int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbDelete(pState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->txn); +} + +SStreamStateCur* streamStateSessionSeekKeyPrev(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + streamStateFreeCur(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToPrev(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + +SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pSessionStateDb, &pCur->pCur, NULL) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c = 0; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateSessionKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + streamStateFreeCur(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + streamStateFreeCur(pCur); + return NULL; + } + + return pCur; +} + +int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } + const SStateSessionKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + return -1; + } + if (pKTmp->opNum != pCur->number) { + return -1; + } + if (pKey->groupId != 0 && pKey->groupId != pKTmp->key.groupId) { + return -1; + } + *pKey = pKTmp->key; + return 0; +} + +int32_t streamStateSessionClear(SStreamState* pState) { + SSessionKey key = {.win.skey = 0, .win.ekey = 0, .groupId = 0}; + streamStateSessionPut(pState, &key, NULL, 0); + SStreamStateCur* pCur = streamStateSessionSeekKeyNext(pState, &key); + while (1) { + SSessionKey delKey = {0}; + void* buf = NULL; + int32_t size = 0; + int32_t code = streamStateSessionGetKVByCur(pCur, &delKey, buf, &size); + if (code == 0) { + memset(buf, 0, size); + streamStateSessionPut(pState, &delKey, buf, size); + } else { + break; + } + streamStateCurNext(pState, pCur); + } + streamStateFreeCur(pCur); + streamStateSessionDel(pState, &key); + return 0; +} + +SStreamStateCur* streamStateSessionGetCur(SStreamState* pState, const SSessionKey* key) { + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + SSessionKey resKey = *key; + while (1) { + streamStateCurPrev(pState, pCur); + SSessionKey tmpKey = *key; + int32_t code = streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); + if (code == TSDB_CODE_SUCCESS && sessionKeyCmpr(key, &tmpKey) == 0) { + resKey = tmpKey; + } else { + break; + } + } + streamStateFreeCur(pCur); + return streamStateSessionGetRanomCur(pState, &resKey); +} + +int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { + // todo refactor + SStreamStateCur* pCur = streamStateSessionGetCur(pState, key); + int32_t size = *pVLen; + void* tmp = NULL; + *pVal = tdbRealloc(NULL, size); + memset(*pVal, 0, size); + if (streamStateSessionGetKVByCur(pCur, key, (const void**)&tmp, pVLen) == 0) { + memcpy(*pVal, tmp, *pVLen); + streamStateFreeCur(pCur); + return 0; + } + streamStateFreeCur(pCur); + return 1; +} + +int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, + state_key_cmpr_fn fn, void** pVal, int32_t* pVLen) { + // todo refactor + int32_t res = TSDB_CODE_SUCCESS; + SSessionKey tmpKey = *key; + int32_t valSize = *pVLen; + void* tmp = tdbRealloc(NULL, valSize); + if (!tmp) { + return -1; + } + + SStreamStateCur* pCur = streamStateSessionGetRanomCur(pState, key); + int32_t code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + memcpy(tmp, *pVal, valSize); + *pVal = tmp; + streamStateFreeCur(pCur); + return res; + } + streamStateFreeCur(pCur); + + streamStateSessionPut(pState, key, NULL, 0); + pCur = streamStateSessionGetRanomCur(pState, key); + streamStateCurPrev(pState, pCur); + code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); + if (fn(pKeyData, stateKey) == true) { + memcpy(tmp, *pVal, valSize); + goto _end; + } + } + + streamStateFreeCur(pCur); + *key = tmpKey; + pCur = streamStateSessionSeekKeyNext(pState, key); + code = streamStateSessionGetKVByCur(pCur, key, (const void**)pVal, pVLen); + if (code == TSDB_CODE_SUCCESS) { + void* stateKey = (char*)(*pVal) + (valSize - keyDataLen); + if (fn(pKeyData, stateKey) == true) { + memcpy(tmp, *pVal, valSize); + goto _end; + } + } + + *key = tmpKey; + res = 1; + memset(tmp, 0, valSize); + +_end: + + *pVal = tmp; + streamStateSessionDel(pState, &tmpKey); + streamStateFreeCur(pCur); + return res; +} diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e000ba8bf8..4a35a15d3e 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -579,6 +579,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -710,6 +711,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc SSyncRaftEntry* pAppendEntry = (SSyncRaftEntry*)(pMsg->data + metaTableArr[i].offset); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -859,6 +861,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -974,6 +977,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index b604d25816..511113352e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -76,7 +76,10 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - ASSERT(pEntry != NULL); + if (pEntry == NULL) { + sError("failed to get entry since %s. index:%lld", tstrerror(terrno), index); + return; + } } // cannot commit, even if quorum agree. need check term! if (pEntry->term <= pSyncNode->pRaftStore->currentTerm) { @@ -127,7 +130,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { // execute fsm if (pSyncNode->pFsm != NULL) { int32_t code = syncNodeCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); - ASSERT(code == 0); + if (code != 0) { + wError("failed to commit sync node since %s", tstrerror(terrno)); + } } } } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 375f2e5730..3f13249ce6 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -96,6 +96,19 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; + } + + if (pSyncNode->replicaNum == 1) { + // only myself, to leader + voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode); + votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode); + + pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum); + + syncNodeCandidate2Leader(pSyncNode); + pSyncNode->pVotesGranted->toLeader = true; + return ret; + } switch (pSyncNode->pRaftCfg->snapshotStrategy) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 9343b45dc7..6e4eaa0aaf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1891,8 +1891,26 @@ inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) { return b1; } +static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg) { + if (pOldCfg->replicaNum != pNewCfg->replicaNum) return true; + if (pOldCfg->myIndex != pNewCfg->myIndex) return true; + for (int32_t i = 0; i < pOldCfg->replicaNum; ++i) { + const SNodeInfo* pOldInfo = &pOldCfg->nodeInfo[i]; + const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; + if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; + if (pOldInfo->nodePort != pNewInfo->nodePort) return true; + } + + return false; +} + void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) { SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg; + if (!syncIsConfigChanged(&oldConfig, pNewConfig)) { + sInfo("vgId:1, sync not reconfig since not changed"); + return; + } + pSyncNode->pRaftCfg->cfg = *pNewConfig; pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex; @@ -2264,7 +2282,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted)); + //ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode, "candidate to leader"); syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); @@ -2657,7 +2675,10 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { if (ths->state == TAOS_SYNC_STATE_LEADER) { int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); + return -1; + } syncNodeReplicate(ths, false); } @@ -2733,7 +2754,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { // del resp mgr, call FpCommitCb - ASSERT(0); + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); return -1; } @@ -2797,8 +2818,8 @@ int32_t syncNodeOnClientRequestBatchCb(SSyncNode* ths, SyncClientRequestBatch* p code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); if (code != 0) { + sError("vgId:%d, failed to append log entry since %s", ths->vgId, tstrerror(terrno)); // del resp mgr, call FpCommitCb - ASSERT(0); return -1; } @@ -3011,7 +3032,8 @@ static int32_t syncNodeProposeConfigChangeFinish(SSyncNode* ths, SyncReconfigFin } bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { - return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); + return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType)); + // return (ths->replicaNum == 1 && syncUtilUserCommit(pMsg->msgType) && ths->vgId != 1); } int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, uint64_t flag) { @@ -3050,7 +3072,10 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); } else { code = ths->pLogStore->syncLogGetEntry(ths->pLogStore, i, &pEntry); - ASSERT(code == 0); + if (code != 0) { + sError("vgId:%d, failed to get log entry since %s. index:%lld", ths->vgId, tstrerror(terrno), i); + return -1; + } ASSERT(pEntry != NULL); } @@ -3060,7 +3085,8 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, // user commit if ((ths->pFsm->FpCommitCb != NULL) && syncUtilUserCommit(pEntry->originalRpcType)) { bool internalExecute = true; - if ((ths->replicaNum == 1) && ths->restoreFinish && ths->vgId != 1) { + if ((ths->replicaNum == 1) && ths->restoreFinish) { + // if ((ths->replicaNum == 1) && ths->restoreFinish && ths->vgId != 1) { internalExecute = false; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 496c8419de..c3dad104d1 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -234,8 +234,6 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); - - ASSERT(0); return -1; } pEntry->index = index; @@ -414,13 +412,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { return 0; } -SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { +SSyncRaftEntry* logStoreGetEntryWithoutLock(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) { - taosThreadMutexLock(&(pData->mutex)); - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReader* pWalHandle = pData->pWalHandle; ASSERT(pWalHandle != NULL); @@ -444,7 +440,8 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } while (0); - ASSERT(0); + sError("failed to read ver since %s. index:%lld", tstrerror(terrno), index); + return NULL; } SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); @@ -465,7 +462,6 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { terrno = saveErr; */ - taosThreadMutexUnlock(&(pData->mutex)); return pEntry; } else { @@ -473,6 +469,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { } } +SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { + SSyncLogStoreData* pData = pLogStore->data; + SSyncRaftEntry *pEntry = NULL; + + taosThreadMutexLock(&pData->mutex); + pEntry = logStoreGetEntryWithoutLock(pLogStore, index); + taosThreadMutexUnlock(&pData->mutex); + return pEntry; +} + int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 7afeb31d4c..8f11fb88a2 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -121,6 +121,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -483,9 +484,8 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN ret = tdbPagerWrite(pPager, pChild); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to write page since %s", terrstr()); + return -1; } // Copy the root page content to the child page @@ -556,8 +556,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pOlds[i]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -583,8 +582,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pParent); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -719,8 +717,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); if (ret < 0) { - // TODO - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } } @@ -937,7 +934,7 @@ static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) // mark dirty ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1009,6 +1006,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { + ASSERT(pVal != NULL && vLen != 0); memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } @@ -1088,6 +1086,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch next ofp, a new ofp and make it dirty ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { + tdbFree(pBuf); return -1; } } @@ -1942,7 +1941,7 @@ int tdbBtcDelete(SBTC *pBtc) { // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -1964,7 +1963,7 @@ int tdbBtcDelete(SBTC *pBtc) { if (idx < nCells) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -2029,7 +2028,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // mark dirty ret = tdbPagerWrite(pBtc->pBt->pPager, pBtc->pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page since %s", terrstr()); return -1; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index ea16e80562..a2803d3ecf 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -106,7 +106,7 @@ int32_t tdbBegin(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to begin pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -121,7 +121,7 @@ int32_t tdbCommit(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerCommit(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to commit pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } @@ -136,7 +136,7 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerAbort(pPager, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("failed to abort pager since %s. dbName:%s, txnId:%ld", tstrerror(terrno), pDb->dbName, pTxn->txnId); return -1; } } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index b6d1f95c0e..732a57639f 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -105,6 +105,7 @@ static int tdbPCacheAlterImpl(SPCache *pCache, int32_t nPage) { for (int32_t iPage = pCache->nPages; iPage < nPage; iPage++) { if (tdbPageCreate(pCache->szPage, &aPage[iPage], tdbDefaultMalloc, NULL) < 0) { // TODO: handle error + tdbOsFree(pCache->aPage); return -1; } @@ -267,7 +268,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) // 4. Try a create new page if (!pPage) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); - if (ret < 0) { + if (ret < 0 || pPage == NULL) { // TODO ASSERT(0); return NULL; @@ -300,8 +301,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) pPage->pPager = pPageH->pPager; memcpy(pPage->pData, pPageH->pData, pPage->pageSize); - tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, - TDB_PAGE_PGNO(pPageH)); + // tdbDebug("pcache/pPageH: %p %d %p %p %d", pPageH, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize, pPage, + // TDB_PAGE_PGNO(pPageH)); tdbPageInit(pPage, pPageH->pPageHdr - pPageH->pData, pPageH->xCellSize); pPage->kLen = pPageH->kLen; pPage->vLen = pPageH->vLen; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 58be8642d7..90332617cd 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -84,7 +84,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { pPager->pCache = pCache; pPager->fd = tdbOsOpen(pPager->dbFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->fd < 0) { + if (TDB_FD_INVALID(pPager->fd)) { + // if (pPager->fd < 0) { return -1; } @@ -156,6 +157,7 @@ int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { + tdbError("failed to write page since %s", terrstr()); return -1; } @@ -210,7 +212,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } } @@ -225,7 +227,9 @@ int tdbPagerBegin(SPager *pPager, TXN *pTxn) { // Open the journal pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (pPager->jfd < 0) { + if (TDB_FD_INVALID(pPager->jfd)) { + tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -243,9 +247,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { // sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } // loop to write the dirty pages to file @@ -255,7 +259,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ret = tdbPagerWritePageToDB(pPager, pPage); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; } } @@ -277,11 +281,25 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // sync the db file - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. file:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } // remove the journal file - tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(pPager->jfd) < 0) { + tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pPager->inTran = 0; return 0; @@ -297,14 +315,14 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 0, sync the journal file ret = tdbOsFSync(pPager->jfd); if (ret < 0) { - // TODO - ASSERT(0); - return 0; + tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { - return 0; + return -1; } ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); @@ -348,7 +366,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { // 4, remove the journal file tdbOsClose(pPager->jfd); - tdbOsRemove(pPager->jFileName); + (void)tdbOsRemove(pPager->jFileName); pPager->inTran = 0; return 0; @@ -516,11 +534,16 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { + tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->jFileName, + pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -540,13 +563,16 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ret = tdbOsWrite(pPager->fd, pPage->pData, pPage->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, + pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -580,33 +606,54 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); if (ret < 0) { + tdbOsFree(pageBuf); return -1; } i64 offset = pPager->pageSize * (pgno - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { - ASSERT(0); + tdbError("failed to lseek fd due to %s. file:%s, offset:%ld", strerror(errno), pPager->dbFileName, offset); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); if (ret < 0) { - ASSERT(0); + tdbError("failed to write buf due to %s. file: %s, bufsize:%d", strerror(errno), pPager->dbFileName, + pPager->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); return -1; } } - tdbOsFSync(pPager->fd); + if (tdbOsFSync(pPager->fd) < 0) { + tdbError("failed to fsync fd due to %s. dbfile:%s", strerror(errno), pPager->dbFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + tdbOsFree(pageBuf); + return -1; + } tdbOsFree(pageBuf); - tdbOsClose(jfd); - tdbOsRemove(pPager->jFileName); + if (tdbOsClose(jfd) < 0) { + tdbError("failed to close jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 380baefb46..008907ca77 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -106,11 +106,13 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF // pTb->pBt ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); if (ret < 0) { + tdbOsFree(pTb); return -1; } ret = tdbPagerRestore(pPager, pTb->pBt); if (ret < 0) { + tdbOsFree(pTb); return -1; } diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 503e109adb..b5dd27052c 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -37,6 +37,8 @@ extern "C" { /* file */ typedef TdFilePtr tdb_fd_t; +#define TDB_FD_INVALID(fd) (fd == NULL) + #define TDB_O_CREAT TD_FILE_CREATE #define TDB_O_WRITE TD_FILE_WRITE #define TDB_O_READ TD_FILE_READ @@ -141,4 +143,4 @@ typedef pthread_mutex_t tdb_mutex_t; } #endif -#endif /*_TDB_OS_H_*/ \ No newline at end of file +#endif /*_TDB_OS_H_*/ diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index c1d8dbef62..2399891f6d 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -146,7 +146,7 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const if (nread < 0) { uError("http-report recv error:%s", uv_err_name(nread)); } else { - uTrace("http-report succ to recv %d bytes", nread); + uTrace("http-report succ to recv %d bytes", (int32_t)nread); } uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c82af0d0e9..237a4b6059 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -54,11 +54,7 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; - if (pInit->connType == TAOS_CONN_SERVER) { - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - } else { - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; uint32_t ip = 0; if (pInit->connType == TAOS_CONN_SERVER) { @@ -79,7 +75,7 @@ void* rpcOpen(const SRpcInit* pInit) { } pRpc->parent = pInit->parent; if (pInit->user) { - memcpy(pRpc->user, pInit->user, strlen(pInit->user)); + memcpy(pRpc->user, pInit->user, TSDB_UNI_LEN); } int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 359b9824cf..48b3097449 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -267,11 +267,12 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) -#define EPSET_FORWARD_INUSE(epSet) \ - do { \ - if ((epSet)->numOfEps != 0) { \ - (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ - } \ +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + if ((epSet)->numOfEps != 0) { \ + ++((epSet)->inUse); \ + (epSet)->inUse = ((epSet)->inUse) % ((epSet)->numOfEps); \ + } \ } while (0) #define EPSET_DEBUG_STR(epSet, tbuf) \ @@ -503,6 +504,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { SConnList list = {0}; taosHashPut((SHashObj*)pool, key, strlen(key), (void*)&list, sizeof(list)); plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + if (plist == NULL) return NULL; QUEUE_INIT(&plist->conns); } @@ -1157,7 +1159,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, SCliObj* cli = taosMemoryCalloc(1, sizeof(SCliObj)); STrans* pTransInst = shandle; - memcpy(cli->label, label, strlen(label)); + memcpy(cli->label, label, TSDB_LABEL_LEN); cli->numOfThreads = numOfThreads; cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*)); @@ -1611,8 +1613,8 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { - memcpy(cvtAddr.ip, ip, strlen(ip)); - memcpy(cvtAddr.fqdn, fqdn, strlen(fqdn)); + if (strlen(ip) <= sizeof(cvtAddr.ip)) memcpy(cvtAddr.ip, ip, strlen(ip)); + if (strlen(fqdn) <= sizeof(cvtAddr.fqdn)) memcpy(cvtAddr.fqdn, fqdn, strlen(fqdn)); cvtAddr.cvt = true; } for (int i = 0; i < pTransInst->numOfThreads; i++) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index f843c8559c..4ae1a1083a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -128,10 +128,10 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { static const int HEADSIZE = sizeof(STransMsgHead); SConnBuffer* p = connBuf; - if (p->left != 0) { + if (p->left != 0 || p->total <= 0) { return -1; } - int total = connBuf->total; + int total = p->total; if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 144cce2cd0..fa22805df2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -35,113 +35,251 @@ int64_t FORCE_INLINE walGetCommittedVer(SWal* pWal) { return pWal->vers.commitVe int64_t FORCE_INLINE walGetAppliedVer(SWal* pWal) { return pWal->vers.appliedVer; } -static FORCE_INLINE void walBuildMetaName(SWal* pWal, int metaVer, char* buf) { - sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); +static FORCE_INLINE int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { + return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); } -static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz > 0); -#if 0 - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - } -#endif +static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { + return sprintf(buf, "%s/meta-ver.tmp", pWal->path); +} - SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); +static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + terrno = TSDB_CODE_SUCCESS; + ASSERT(fileIdx >= 0 && fileIdx < sz); + + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); - int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); - pLastFileInfo->fileSize = fileSize; - TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); if (pFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + // ensure size as non-negative + pFileInfo->fileSize = TMAX(0, pFileInfo->fileSize); + uint64_t magic = WAL_MAGIC; + int64_t walCkHeadSz = sizeof(SWalCkHead); + int64_t end = fileSize; + int64_t offset = 0; + int32_t capacity = 0; + int32_t readSize = 0; + char* buf = NULL; + char* found = NULL; + bool firstTrial = pFileInfo->fileSize < fileSize; - char* buf = taosMemoryMalloc(readSize + 5); - if (buf == NULL) { - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; - return -1; - } - - int64_t offset; - offset = taosLSeekFile(pFile, -readSize, SEEK_END); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - char* found = NULL; + // search for the valid last WAL entry, e.g. block by block while (1) { + offset = (firstTrial) ? pFileInfo->fileSize : TMAX(0, end - WAL_SCAN_BUF_SIZE); + ASSERT(offset <= end); + readSize = end - offset; + capacity = readSize + sizeof(magic); + + int64_t limit = WAL_RECOV_SIZE_LIMIT; + if (limit < readSize) { + wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64 + ", end:%" PRId64 ", file:%s", + pWal->cfg.vgId, limit, offset, end, fnameStr); + terrno = TSDB_CODE_WAL_SIZE_LIMIT; + goto _err; + } + + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + + int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), offset); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (readSize != taosReadFile(pFile, buf, readSize)) { + wError("vgId:%d, failed to read file due to %s. readSize:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + readSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + char* candidate = NULL; char* haystack = buf; - char* candidate; - while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { - // read and validate - SWalCkHead* logContent = (SWalCkHead*)candidate; - if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { - found = candidate; + + while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(magic))) != NULL) { + // validate head + int64_t len = readSize - (candidate - buf); + if (len < walCkHeadSz) { + break; } + SWalCkHead* logContent = (SWalCkHead*)candidate; + if (walValidHeadCksum(logContent) != 0) { + wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // validate body + int64_t size = walCkHeadSz + logContent->head.bodyLen; + if (len < size) { + int64_t extraSize = size - len; + if (capacity < readSize + extraSize + sizeof(magic)) { + capacity += extraSize; + void* ptr = taosMemoryRealloc(buf, capacity); + if (ptr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + buf = ptr; + } + int64_t ret = taosLSeekFile(pFile, offset + readSize, SEEK_SET); + if (ret < 0) { + wError("vgId:%d, failed to lseek file due to %s. offset:%" PRId64 "", pWal->cfg.vgId, strerror(errno), + offset); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + if (extraSize != taosReadFile(pFile, buf + readSize, extraSize)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", extraSize:%" PRId64 ", file:%s", + pWal->cfg.vgId, strerror(errno), offset + readSize, extraSize, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + break; + } + } + if (walValidBodyCksum(logContent) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); + haystack = candidate + 1; + if (firstTrial) { + break; + } else { + continue; + } + } + + // found one + found = candidate; haystack = candidate + 1; } + if (found || offset == 0) break; - offset = TMIN(0, offset - readSize + sizeof(uint64_t)); - int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); - ASSERT(offset == offset2); - if (readSize != taosReadFile(pFile, buf, readSize)) { - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } -#if 0 - if (found == buf) { - SWalCkHead* logContent = (SWalCkHead*)found; - if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { - // file has to be deleted - taosMemoryFree(buf); - taosCloseFile(&pFile); - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; - } - } -#endif + + // go backwards, e.g. by at most one WAL scan buf size + end = offset + walCkHeadSz - 1; + firstTrial = false; } - if (found == NULL) { - // file corrupted, no complete log - // TODO delete and search in previous files - /*ASSERT(0);*/ - terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - return -1; + // determine end of last entry + SWalCkHead* lastEntry = (SWalCkHead*)found; + int64_t retVer = -1; + int64_t lastEntryBeginOffset = 0; + int64_t lastEntryEndOffset = 0; + + if (lastEntry == NULL) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + } else { + retVer = lastEntry->head.version; + lastEntryBeginOffset = offset + (int64_t)((char*)lastEntry - (char*)buf); + lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; } // truncate file - SWalCkHead* lastEntry = (SWalCkHead*)found; - int64_t retVer = lastEntry->head.version; - int64_t lastEntryBeginOffset = offset + (int64_t)((char*)found - (char*)buf); - int64_t lastEntryEndOffset = lastEntryBeginOffset + sizeof(SWalCkHead) + lastEntry->head.bodyLen; if (lastEntryEndOffset != fileSize) { - wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, + wWarn("vgId:%d, repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, fileSize); - taosFtruncateFile(pFile, lastEntryEndOffset); - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; - pWal->totSize -= (fileSize - lastEntryEndOffset); + if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) { + wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (taosFsyncFile(pFile) < 0) { + wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } + pFileInfo->fileSize = lastEntryEndOffset; taosCloseFile(&pFile); taosMemoryFree(buf); - return retVer; + +_err: + taosCloseFile(&pFile); + taosMemoryFree(buf); + return -1; +} + +static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { + int metaFileNum = taosArrayGetSize(metaLogList); + int actualFileNum = taosArrayGetSize(actualLogList); + int j = 0; + + // both of the lists in asc order + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); + while (j < metaFileNum) { + SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); + ASSERT(pMetaInfo != NULL); + if (pMetaInfo->firstVer < pLogInfo->firstVer) { + j++; + } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { + (*pLogInfo) = *pMetaInfo; + j++; + break; + } else { + break; + } + } + } + + taosArrayClear(metaLogList); + + for (int i = 0; i < actualFileNum; i++) { + SWalFileInfo* pFileInfo = taosArrayGet(actualLogList, i); + taosArrayPush(metaLogList, pFileInfo); + } +} + +void walAlignVersions(SWal* pWal) { + if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { + wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, + pWal->vers.firstVer, pWal->vers.snapshotVer); + pWal->vers.firstVer = pWal->vers.snapshotVer + 1; + } + if (pWal->vers.lastVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, lastVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.lastVer, pWal->vers.snapshotVer); + pWal->vers.lastVer = pWal->vers.snapshotVer; + } + if (pWal->vers.commitVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, commitVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.commitVer, pWal->vers.snapshotVer); + pWal->vers.commitVer = pWal->vers.snapshotVer; + } + if (pWal->vers.appliedVer < pWal->vers.snapshotVer) { + wWarn("vgId:%d, appliedVer:%" PRId64 " is less than snapshotVer:%" PRId64 ". align with it.", pWal->cfg.vgId, + pWal->vers.appliedVer, pWal->vers.snapshotVer); + pWal->vers.appliedVer = pWal->vers.snapshotVer; + } + + pWal->vers.commitVer = TMIN(pWal->vers.lastVer, pWal->vers.commitVer); + pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); } int walCheckAndRepairMeta(SWal* pWal) { @@ -150,7 +288,6 @@ int walCheckAndRepairMeta(SWal* pWal) { const char* idxPattern = "^[0-9]+.idx$"; regex_t logRegPattern; regex_t idxRegPattern; - bool fixed = false; regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); @@ -184,225 +321,243 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArraySort(actualLog, compareWalFileInfo); - int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); - int actualFileNum = taosArrayGetSize(actualLog); + int metaFileNum = taosArrayGetSize(pWal->fileInfoSet); + int actualFileNum = taosArrayGetSize(actualLog); + int64_t firstVerPrev = pWal->vers.firstVer; + int64_t lastVerPrev = pWal->vers.lastVer; + int64_t totSize = 0; + bool updateMeta = (metaFileNum != actualFileNum); -#if 0 - for (int32_t fileNo = actualFileNum - 1; fileNo >= 0; fileNo--) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, fileNo); + // rebuild meta of file info + walRebuildFileInfoSet(pWal->fileInfoSet, actualLog); + taosArrayDestroy(actualLog); + + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + ASSERT(sz == actualFileNum); + + // scan and determine the lastVer + int32_t fileIdx = sz; + + while (--fileIdx >= 0) { char fnameStr[WAL_FILE_LEN]; + int64_t fileSize = 0; + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fileSize = 0; - taosStatFile(fnameStr, &fileSize, NULL); - if (fileSize == 0) { + int32_t code = taosStatFile(fnameStr, &fileSize, NULL); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); + return -1; + } + + ASSERT(pFileInfo->firstVer >= 0); + + if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { + totSize += pFileInfo->fileSize; + continue; + } + updateMeta = true; + + int64_t lastVer = walScanLogGetLastVer(pWal, fileIdx); + if (lastVer < 0) { + if (terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) { + wError("failed to scan wal last ver since %s", terrstr()); + return -1; + } + ASSERT(pFileInfo->fileSize == 0); + // remove the empty wal log, and its idx taosRemoveFile(fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); - taosArrayPop(pLogInfoArray); - } else { - break; + // remove its meta entry + taosArrayRemove(pWal->fileInfoSet, fileIdx); + continue; } + + // update lastVer + pFileInfo->lastVer = lastVer; + totSize += pFileInfo->fileSize; } - actualFileNum = taosArrayGetSize(pLogInfoArray); -#endif - - { - int32_t i = 0, j = 0; - while (i < actualFileNum && j < metaFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - SWalFileInfo* pMetaFile = taosArrayGet(pWal->fileInfoSet, j); - if (pActualFile->firstVer < pMetaFile->firstVer) { - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - i++; - } else if (pActualFile->firstVer > pMetaFile->firstVer) { - taosArrayRemove(pWal->fileInfoSet, j); - metaFileNum--; - } else { - i++; - j++; - } - } - if (i == actualFileNum && j == metaFileNum) { - if (j > 0) { - SWalFileInfo* pLastInfo = taosArrayGet(pWal->fileInfoSet, j - 1); - int64_t fsize = 0; - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pLastInfo->firstVer, fNameStr); - taosStatFile(fNameStr, &fsize, NULL); - if (pLastInfo->fileSize != fsize) { - fixed = true; - pLastInfo->fileSize = fsize; - pLastInfo->lastVer = walScanLogGetLastVer(pWal); - } - } - } else { - fixed = true; - while (i < actualFileNum) { - SWalFileInfo* pActualFile = taosArrayGet(actualLog, i); - char fNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pActualFile->firstVer, fNameStr); - taosStatFile(fNameStr, &pActualFile->fileSize, NULL); - - if (pActualFile->fileSize == 0) { - ASSERT(i == actualFileNum - 1); - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - break; - } - - if (i < actualFileNum - 1) { - pActualFile->lastVer = ((SWalFileInfo*)taosArrayGet(actualLog, i + 1))->firstVer - 1; - taosArrayPush(pWal->fileInfoSet, pActualFile); - i++; - } else { - pActualFile = taosArrayPush(pWal->fileInfoSet, pActualFile); - pActualFile->lastVer = walScanLogGetLastVer(pWal); - if (pActualFile->lastVer == -1) { - taosRemoveFile(fNameStr); - - walBuildIdxName(pWal, pActualFile->firstVer, fNameStr); - taosRemoveFile(fNameStr); - taosArrayPop(pWal->fileInfoSet); - } - break; - } - } - } - } - -#if 0 - if (metaFileNum > actualFileNum) { - taosArrayPopFrontBatch(pWal->fileInfoSet, metaFileNum - actualFileNum); - } else if (metaFileNum < actualFileNum) { - for (int i = metaFileNum; i < actualFileNum; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(actualLog, i); - taosArrayPush(pWal->fileInfoSet, pFileInfo); - } - } -#endif - - taosArrayDestroy(actualLog); - + // reset vers info and so on actualFileNum = taosArrayGetSize(pWal->fileInfoSet); pWal->writeCur = actualFileNum - 1; - + pWal->totSize = totSize; + pWal->vers.lastVer = -1; if (actualFileNum > 0) { - int64_t fLastVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur))->lastVer; - if (fLastVer != -1 && pWal->vers.lastVer != fLastVer) { - fixed = true; - pWal->vers.lastVer = fLastVer; - } - int64_t fFirstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; - if (fFirstVer != pWal->vers.firstVer) { - fixed = true; - pWal->vers.firstVer = fFirstVer; - } + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; + } + (void)walAlignVersions(pWal); + + // update meta file + if (updateMeta) { + (void)walSaveMeta(pWal); + } + return 0; +} + +int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { + if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - if (fixed) { - walSaveMeta(pWal); + if (taosReadFile(pLogFile, pCkHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (walValidHeadCksum(pCkHead) != 0) { + terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; + return -1; } return 0; } -int walCheckAndRepairIdx(SWal* pWal) { +int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - for (int32_t i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); + ASSERT(fileIdx >= 0 && fileIdx < sz); + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + char fnameStr[WAL_FILE_LEN]; + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + char fLogNameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); + int64_t fileSize = 0; - char fnameStr[WAL_FILE_LEN]; - walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); - int64_t fsize; - TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); - if (pIdxFile == NULL) { - ASSERT(0); + if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); + if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { + return 0; + } + + // start to repair + int64_t offset = fileSize - fileSize % sizeof(SWalIdxEntry); + TdFilePtr pLogFile = NULL; + TdFilePtr pIdxFile = NULL; + SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer - 1, .offset = -sizeof(SWalCkHead)}; + SWalCkHead ckHead; + memset(&ckHead, 0, sizeof(ckHead)); + ckHead.head.version = idxEntry.ver; + + pIdxFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE); + if (pIdxFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); + if (pLogFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); + goto _err; + } + + // determine the last valid entry end, i.e. offset + while ((offset -= sizeof(SWalIdxEntry)) >= 0) { + if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fnameStr, terrstr()); - return -1; + goto _err; } - taosFStatFile(pIdxFile, &fsize, NULL); - if (fsize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { - taosCloseFile(&pIdxFile); + if (taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to read file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (idxEntry.ver > pFileInfo->lastVer) { continue; } - int32_t left = fsize % sizeof(SWalIdxEntry); - int64_t offset = taosLSeekFile(pIdxFile, -left, SEEK_END); - if (left != 0) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since size invalid, file size %ld", pWal->cfg.vgId, fnameStr, - offset, fsize); - } - offset -= sizeof(SWalIdxEntry); - - SWalIdxEntry idxEntry = {.ver = pFileInfo->firstVer}; - while (1) { - if (offset < 0) { - taosLSeekFile(pIdxFile, 0, SEEK_SET); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - break; - } - if (taosLSeekFile(pIdxFile, offset, SEEK_SET) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d cannot seek offset %ld when repair idx since %s", pWal->cfg.vgId, offset, terrstr()); - } - int64_t contLen = taosReadFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - if (contLen < 0 || contLen != sizeof(SWalIdxEntry)) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if ((idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry) != offset) { - taosFtruncateFile(pIdxFile, offset); - wWarn("vgId:%d wal truncate file %s to offset %ld since entry invalid, entry ver %ld, entry offset %ld", - pWal->cfg.vgId, fnameStr, offset, idxEntry.ver, idxEntry.offset); - offset -= sizeof(SWalIdxEntry); - } else { - break; - } + if (offset != (idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry)) { + continue; } - if (idxEntry.ver < pFileInfo->lastVer) { - char fLogNameStr[WAL_FILE_LEN]; - walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); - TdFilePtr pLogFile = taosOpenFile(fLogNameStr, TD_FILE_READ); - if (pLogFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot open file %s, since %s", pWal->cfg.vgId, fLogNameStr, terrstr()); - return -1; - } - while (idxEntry.ver < pFileInfo->lastVer) { - if (taosLSeekFile(pLogFile, idxEntry.offset, SEEK_SET) == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, cannot seek file %s at %ld, since %s", pWal->cfg.vgId, fLogNameStr, idxEntry.offset, - terrstr()); - return -1; - } - SWalCkHead ckHead; - taosReadFile(pLogFile, &ckHead, sizeof(SWalCkHead)); - if (idxEntry.ver != ckHead.head.version) { - // todo truncate this idx also - taosCloseFile(&pLogFile); - wError("vgId:%d, invalid repair case, log seek to %ld to find ver %ld, actual ver %ld", pWal->cfg.vgId, - idxEntry.offset, idxEntry.ver, ckHead.head.version); - return -1; - } - idxEntry.ver = ckHead.head.version + 1; - idxEntry.offset = idxEntry.offset + sizeof(SWalCkHead) + ckHead.head.bodyLen; - wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); - taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)); - } - taosCloseFile(&pLogFile); + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", + pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); + continue; + } + + if (idxEntry.ver == ckHead.head.version) { + break; + } + } + offset += sizeof(SWalIdxEntry); + + ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); + + // ftruncate idx file + if (offset < fileSize) { + if (taosFtruncateFile(pIdxFile, offset) < 0) { + wError("vgId:%d, failed to ftruncate file due to %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + strerror(errno), offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } + + // rebuild idx file + if (taosLSeekFile(pIdxFile, 0, SEEK_END) < 0) { + wError("vgId:%d, failed to seek file due to %s. offset:" PRId64 ", file:%s", pWal->cfg.vgId, strerror(errno), + offset, fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + while (idxEntry.ver < pFileInfo->lastVer) { + ASSERT(idxEntry.ver == ckHead.head.version); + + idxEntry.ver += 1; + idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; + + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { + wError("vgId:%d, failed to read wal log head since %s. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, terrstr(), + idxEntry.offset, fLogNameStr); + goto _err; + } + wWarn("vgId:%d wal idx append new entry %ld %ld", pWal->cfg.vgId, idxEntry.ver, idxEntry.offset); + if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + } + + if (taosFsyncFile(pIdxFile) < 0) { + wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); + goto _err; + } + + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return 0; + +_err: + (void)taosCloseFile(&pLogFile); + (void)taosCloseFile(&pIdxFile); + return -1; +} + +int walCheckAndRepairIdx(SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + int32_t fileIdx = sz; + while (--fileIdx >= 0) { + if (walCheckAndRepairIdxFile(pWal, fileIdx) < 0) { + wError("vgId:%d, failed to repair idx file since %s. fileIdx:%d", pWal->cfg.vgId, terrstr(), fileIdx); + return -1; } - taosCloseFile(&pIdxFile); } return 0; } @@ -495,14 +650,20 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); + if (!pRoot) goto _err; pMeta = cJSON_GetObjectItem(pRoot, "meta"); + if (!pMeta) goto _err; pField = cJSON_GetObjectItem(pMeta, "firstVer"); + if (!pField) goto _err; pWal->vers.firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "snapshotVer"); + if (!pField) goto _err; pWal->vers.snapshotVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "commitVer"); + if (!pField) goto _err; pWal->vers.commitVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pMeta, "lastVer"); + if (!pField) goto _err; pWal->vers.lastVer = atoll(cJSON_GetStringValue(pField)); pFiles = cJSON_GetObjectItem(pRoot, "files"); @@ -512,17 +673,23 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { taosArrayEnsureCap(pArray, sz); SWalFileInfo* pData = pArray->pData; for (int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + if (!pInfoJson) goto _err; SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); + if (!pField) goto _err; pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); + if (!pField) goto _err; pInfo->lastVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "createTs"); + if (!pField) goto _err; pInfo->createTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "closeTs"); + if (!pField) goto _err; pInfo->closeTs = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "fileSize"); + if (!pField) goto _err; pInfo->fileSize = atoll(cJSON_GetStringValue(pField)); } taosArraySetSize(pArray, sz); @@ -530,6 +697,10 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pWal->writeCur = sz - 1; cJSON_Delete(pRoot); return 0; + +_err: + cJSON_Delete(pRoot); + return -1; } static int walFindCurMetaVer(SWal* pWal) { @@ -565,22 +736,63 @@ static int walFindCurMetaVer(SWal* pWal) { int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer + 1, fnameStr); - TdFilePtr pMetaFile = taosOpenFile(fnameStr, TD_FILE_CREATE | TD_FILE_WRITE); - if (pMetaFile == NULL) { + char tmpFnameStr[WAL_FILE_LEN]; + int n; + + // fsync the idx and log file at first to ensure validity of meta + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + + if (taosFsyncFile(pWal->pLogFile) < 0) { + wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + // flush to a tmpfile + n = walBuildTmpMetaName(pWal, tmpFnameStr); + ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + + TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pMetaFile == NULL) { + wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); if (len != taosWriteFile(pMetaFile, serialized, len)) { - // TODO:clean file - - taosCloseFile(&pMetaFile); - taosRemoveFile(fnameStr); - return -1; + wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosFsyncFile(pMetaFile) < 0) { + wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (taosCloseFile(&pMetaFile) < 0) { + wError("vgId:%d, failed to close file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // rename it + n = walBuildMetaName(pWal, metaVer + 1, fnameStr); + ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + + if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { + wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } - taosCloseFile(&pMetaFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -588,6 +800,11 @@ int walSaveMeta(SWal* pWal) { } taosMemoryFree(serialized); return 0; + +_err: + taosCloseFile(&pMetaFile); + taosMemoryFree(serialized); + return -1; } int walLoadMeta(SWal* pWal) { @@ -629,6 +846,10 @@ int walLoadMeta(SWal* pWal) { } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); + if (code < 0) { + wError("failed to deserialize wal meta. file:%s", fnameStr); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + } taosCloseFile(&pFile); taosMemoryFree(buf); return code; diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 7974f3e32e..29058cada1 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -81,6 +81,12 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } + if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(pWal); + return NULL; + } + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); @@ -98,15 +104,14 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { tstrncpy(pWal->path, path, sizeof(pWal->path)); if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init ref pWal->pRefHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pWal->pRefHash == NULL) { - taosMemoryFree(pWal); - return NULL; + wError("failed to init hash since %s", tstrerror(terrno)); + goto _err; } // open meta @@ -117,9 +122,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; + goto _err; } // init status @@ -131,46 +134,37 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeHead.head.protoVer = WAL_PROTO_VER; pWal->writeHead.magic = WAL_MAGIC; - if (taosThreadMutexInit(&pWal->mutex, NULL) < 0) { - taosArrayDestroy(pWal->fileInfoSet); - taosHashCleanup(pWal->pRefHash); - taosMemoryFree(pWal); - return NULL; - } - - pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if (pWal->refId < 0) { - taosHashCleanup(pWal->pRefHash); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - taosMemoryFree(pWal); - return NULL; - } - - walLoadMeta(pWal); + // load meta + (void)walLoadMeta(pWal); if (walCheckAndRepairMeta(pWal) < 0) { wError("vgId:%d cannot open wal since repair meta file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; } if (walCheckAndRepairIdx(pWal) < 0) { wError("vgId:%d cannot open wal since repair idx file failed", pWal->cfg.vgId); - taosHashCleanup(pWal->pRefHash); - taosRemoveRef(tsWal.refSetId, pWal->refId); - taosThreadMutexDestroy(&pWal->mutex); - taosArrayDestroy(pWal->fileInfoSet); - return NULL; + goto _err; + } + + // add ref + pWal->refId = taosAddRef(tsWal.refSetId, pWal); + if (pWal->refId < 0) { + wError("failed to add ref for Wal since %s", tstrerror(terrno)); + goto _err; } wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); - return pWal; + +_err: + taosArrayDestroy(pWal->fileInfoSet); + taosHashCleanup(pWal->pRefHash); + taosThreadMutexDestroy(&pWal->mutex); + taosMemoryFree(pWal); + pWal = NULL; + return NULL; } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { @@ -195,11 +189,11 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { taosThreadMutexLock(&pWal->mutex); + (void)walSaveMeta(pWal); taosCloseFile(&pWal->pLogFile); pWal->pLogFile = NULL; taosCloseFile(&pWal->pIdxFile); pWal->pIdxFile = NULL; - walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; taosHashCleanup(pWal->pRefHash); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b2cd4fac11..179d809c84 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -181,7 +181,11 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(pRet != NULL); + if (pRet == NULL) { + wError("failed to find WAL log file with ver:%lld", ver); + terrno = TSDB_CODE_WAL_INVALID_VER; + return -1; + } if (pReader->curFileFirstVer != pRet->firstVer) { // error code was set inner if (walReadChangeFile(pReader, pRet->firstVer) < 0) { @@ -472,7 +476,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - ASSERT(0); + wError("vgId:%d, failed to read WAL record head, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } @@ -505,6 +510,8 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + wError("vgId:%d, failed to read WAL record body, index:%" PRId64 ", from log file since %s", + pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 1196914dae..4b75db52b7 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,6 +79,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { + code = taosFsyncFile(pWal->pLogFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pLogFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -86,6 +91,11 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { + code = taosFsyncFile(pWal->pIdxFile); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } code = taosCloseFile(&pWal->pIdxFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index e7161079d9..0562bbad27 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -45,6 +45,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d restore from snapshot, cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); @@ -53,6 +54,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { if (taosRemoveFile(fnameStr) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d cannot remove file %s since %s", pWal->cfg.vgId, fnameStr, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); return -1; } wInfo("vgId:%d restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); @@ -219,10 +221,12 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); - taosFsyncFile(pWal->pLogFile); - taosFsyncFile(pWal->pIdxFile); - - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + taosThreadMutexUnlock(&pWal->mutex); + return -1; + } // unlock taosThreadMutexUnlock(&pWal->mutex); @@ -394,7 +398,11 @@ int32_t walRollImpl(SWal *pWal) { pWal->lastRollSeq = walGetSeq(); - walSaveMeta(pWal); + code = walSaveMeta(pWal); + if (code < 0) { + wError("vgId:%d, failed to save meta since %s", pWal->cfg.vgId, terrstr()); + goto END; + } END: return code; @@ -402,25 +410,35 @@ END: static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int64_t idxOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + ASSERT(pFileInfo->firstVer >= 0); + int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, idxOffset); + int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to write idx entry due to %s. ver:%lld", pWal->cfg.vgId, strerror(errno), ver); terrno = TAOS_SYSTEM_ERROR(errno); - // TODO truncate return -1; } + + ASSERT(taosLSeekFile(pWal->pIdxFile, 0, SEEK_END) == idxOffset + sizeof(SWalIdxEntry) && "Offset of idx entries misaligned"); return 0; } -// TODO gurantee atomicity by truncate failed writing static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen) { int64_t code = 0; int64_t offset = walGetCurFileOffset(pWal); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + if (pFileInfo->firstVer == -1) { + pFileInfo->firstVer = index; + } pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -431,11 +449,14 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + code = walWriteIndex(pWal, index, offset); + if (code < 0) { + goto END; + } + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -444,7 +465,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -452,24 +472,31 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy goto END; } - code = walWriteIndex(pWal, index, offset); - if (code < 0) { - // TODO ftruncate - goto END; - } - // set status if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + bodyLen; - if (walGetCurFileInfo(pWal)->firstVer == -1) { - walGetCurFileInfo(pWal)->firstVer = index; - } - walGetCurFileInfo(pWal)->lastVer = index; - walGetCurFileInfo(pWal)->fileSize += sizeof(SWalCkHead) + bodyLen; + pFileInfo->lastVer = index; + pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen; return 0; + END: + // recover in a reverse order + if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { + wFatal("vgId:%d, failed to ftruncate logfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, offset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } + + int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); + if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { + wFatal("vgId:%d, failed to ftruncate idxfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, idxOffset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } return -1; } @@ -550,6 +577,11 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { + wTrace("vgId:%d, fileId:%" PRId64 ".idx, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + if (taosFsyncFile(pWal->pIdxFile) < 0) { + wError("vgId:%d, file:%" PRId64 ".idx, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); + } wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index f0442c6fd1..616ab7875d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -87,11 +87,17 @@ void osUpdate() { void osCleanup() {} -bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } +bool osLogSpaceAvailable() { return tsLogSpace.size.avail > 0; } -bool osDataSpaceAvailable() { return tsDataSpace.reserved <= tsDataSpace.size.avail; } +bool osDataSpaceAvailable() { return tsDataSpace.size.avail > 0; } -bool osTempSpaceAvailable() { return tsTempSpace.reserved <= tsTempSpace.size.avail; } +bool osTempSpaceAvailable() { return tsTempSpace.size.avail > 0; } + +bool osLogSpaceSufficient() { return tsLogSpace.size.avail > tsLogSpace.reserved; } + +bool osDataSpaceSufficient() { return tsDataSpace.size.avail > tsDataSpace.reserved; } + +bool osTempSpaceSufficient() { return tsTempSpace.size.avail > tsTempSpace.reserved; } void osSetTimezone(const char *timezone) { taosSetSystemTimezone(timezone, tsTimezoneStr, &tsDaylight, &tsTimezone); } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index bfeef248cd..fc943d4d76 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -590,7 +590,9 @@ TdSocketPtr taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t taosCloseSocket(&pSocket); return NULL; } else { - taosKeepTcpAlive(pSocket); + if (taosKeepTcpAlive(pSocket) == -1) { + return NULL; + } } return pSocket; @@ -1059,18 +1061,22 @@ int32_t taosCreateSocketWithTimeout(uint32_t timeout) { } #if defined(WINDOWS) if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { + taosCloseSocketNoCheck1(fd); return -1; } #elif defined(_TD_DARWIN_64) uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_CONNECTIONTIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + taosCloseSocketNoCheck1(fd); return -1; } #else // Linux like systems uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { + taosCloseSocketNoCheck1(fd); return -1; } #endif + return (int)fd; } diff --git a/source/util/src/tbloomfilter.c b/source/util/src/tbloomfilter.c index 7e1506c140..f3ccbb0aac 100644 --- a/source/util/src/tbloomfilter.c +++ b/source/util/src/tbloomfilter.c @@ -19,12 +19,12 @@ #include "taos.h" #include "taoserror.h" -#define UNIT_NUM_BITS 64 -#define UNIT_ADDR_NUM_BITS 6 +#define UNIT_NUM_BITS 64ULL +#define UNIT_ADDR_NUM_BITS 6ULL static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); uint64_t old = buf[unitIndex]; buf[unitIndex] |= mask; return buf[unitIndex] != old; @@ -32,7 +32,7 @@ static FORCE_INLINE bool setBit(uint64_t *buf, uint64_t index) { static FORCE_INLINE bool getBit(uint64_t *buf, uint64_t index) { uint64_t unitIndex = index >> UNIT_ADDR_NUM_BITS; - uint64_t mask = 1 << (index % UNIT_NUM_BITS); + uint64_t mask = 1ULL << (index % UNIT_NUM_BITS); return buf[unitIndex] & mask; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index eb13a08be4..37f9734eab 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -238,8 +238,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_ALREADY_EXIST, "Qnode already exists" TAOS_DEFINE_ERROR(TSDB_CODE_MND_QNODE_NOT_EXIST, "Qnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_ALREADY_EXIST, "Snode already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SNODE_NOT_EXIST, "Snode not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_ALREADY_EXIST, "Bnode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BNODE_NOT_EXIST, "Bnode not there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_FEW_MNODES, "The replica of mnode cannot less than 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_MNODES, "The replica of mnode cannot exceed 3") @@ -447,12 +445,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Wal unexpected generic error") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2e2300ba14..9d97cf7ab2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -443,10 +443,13 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { taosUpdateLogNums(level); - if (tsAsyncLog) { + if (tsAsyncLog && level != DEBUG_FATAL) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); + if (level == DEBUG_FATAL) { + taosFsyncFile(tsLogObj.logHandle->pFile); + } } if (tsLogObj.maxLines > 0) { diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 9cf9e2c431..f524680331 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -38,6 +38,7 @@ void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *lab if (pSched->queue == NULL) { uError("%s: no enough memory for queue", label); taosCleanUpScheduler(pSched); + taosMemoryFree(pSched); return NULL; } diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 58dcb87345..e0b905375a 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -67,9 +67,13 @@ mkdir -p /var/lib/taos cd $CONTAINER_TESTDIR/tests/$exec_dir ulimit -c unlimited +md5sum /usr/lib/libtaos.so.1 +md5sum /home/TDinternal/debug/build/lib/libtaos.so $TIMEOUT_CMD $cmd RET=$? echo "cmd exit code: $RET" +md5sum /usr/lib/libtaos.so.1 +md5sum /home/TDinternal/debug/build/lib/libtaos.so if [ $RET -ne 0 ]; then pwd diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index bb57f238f0..f58aaaf29d 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -68,12 +68,16 @@ if [ $ent -ne 0 ]; then CONTAINER_TESTDIR=/home/TDinternal/community SIM_DIR=/home/TDinternal/sim REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" + REP_MOUNT_LIB="$INTERNAL_REPDIR/debug/build/lib:/home/TDinternal/debug/build/lib:ro" + else # community edition REPDIR=$WORKDIR/TDengine CONTAINER_TESTDIR=/home/TDengine SIM_DIR=/home/TDengine/sim REP_MOUNT_PARAM="$REPDIR:/home/TDengine" + REP_MOUNT_LIB="$REPDIR/debug/build/lib:/home/TDengine/debug/build/lib:ro" + fi ulimit -c unlimited @@ -103,6 +107,7 @@ coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` docker run \ -v $REP_MOUNT_PARAM \ + -v $REP_MOUNT_LIB \ -v $MOUNT_DIR \ -v ${SOURCEDIR}:/usr/local/src/ \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh new file mode 100755 index 0000000000..457c9eae20 --- /dev/null +++ b/tests/script/coverage_test.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +branch= +if [ x$1 != x ];then + branch=$1 + echo "Testing branch: $branch" +else + echo "Please enter branch name as a parameter" + exit 1 +fi + +today=`date +"%Y%m%d"` +TDENGINE_DIR=/root/pxiao/TDengine +TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function buildTDengine { + echo "check if TDengine need build" + cd $TDENGINE_DIR + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + + # reset counter + lcov -d . --zerocounters + + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git clean -dfx + git pull + git submodule update --init --recursive -f + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + echo "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + if [ "$branch" == "3.0" ]; then + echo "3.0 =============" + cmake -DCOVER=true -DBUILD_TEST=true .. + else + cmake -DCOVER=true -DBUILD_TOOLS=true -DBUILD_HTTP=false .. > /dev/null + fi + make -j4 + make install +} + +function runGeneralCaseOneByOne { + while read -r line; do + if [[ $line =~ ^./test.sh* ]]; then + case=`echo $line | grep sim$ | awk '{print $NF}'` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTestNGCaseOneByOne { + while read -r line; do + if [[ $line =~ ^taostest* ]]; then + case=`echo $line | cut -d' ' -f 3 | cut -d'=' -f 2` + yaml=`echo $line | cut -d' ' -f 2` + + if [ -n "$case" ]; then + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && taostest $yaml --case=$case --keep --disable_collection > /dev/null 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + fi + fi + done < $1 +} + +function runTest { + echo "run Test" + + if [ "$branch" == "3.0" ]; then + echo "start run unit test case ................" + echo " $TDENGINE_DIR/debug " + cd $TDENGINE_DIR/debug + ctest -j12 + echo "3.0 unit test done" + fi + + cd $TDENGINE_DIR/tests/script + + [ -d ../../sim ] && rm -rf ../../sim + [ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT + + runGeneralCaseOneByOne jenkins/basic.txt + + sed -i "1i\SIM cases test result" $TDENGINE_COVERAGE_REPORT + + totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + sed -i -e "2i\ ### Total $totalSuccess SIM test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + sed -i "3i\### Total $totalFailed SIM test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + sed "3G" $TDENGINE_COVERAGE_REPORT + + stopTaosd + echo "run TestNG cases" + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + cd $TDENGINE_DIR/../TestNG/cases + runTestNGCaseOneByOne ../scripts/cases.txt + echo "TestNG cases done" + + cd $TDENGINE_DIR/tests + rm -rf ../sim + /root/pxiao/test-all-coverage.sh full python $branch | tee -a $TDENGINE_COVERAGE_REPORT + + + sed -i "4i\Python cases test result" $TDENGINE_COVERAGE_REPORT + totalPySuccess=`grep 'python case(s) succeed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ "$totalPySuccess" -gt "0" ]; then + sed -i -e "5i\ ### Total $totalPySuccess Python test case(s) succeed! ###" $TDENGINE_COVERAGE_REPORT + fi + + totalPyFailed=`grep 'python case(s) failed!' $TDENGINE_COVERAGE_REPORT | awk '{print $4}'` + if [ -z $totalPyFailed ]; then + sed -i "6i\\n" $TDENGINE_COVERAGE_REPORT + else + sed -i "6i\### Total $totalPyFailed Python test case(s) failed! ###" $TDENGINE_COVERAGE_REPORT + fi + + echo "### run JDBC test cases ###" | tee -a $TDENGINE_COVERAGE_REPORT + # Test Connector + stopTaosd + nohup $TDENGINE_DIR/debug/build/bin/taosd -c /etc/taos > /dev/null 2>&1 & + sleep 10 + + cd $TDENGINE_DIR/src/connector/jdbc + mvn clean package > /dev/null 2>&1 + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test C Demo + stopTaosd + $TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null & + sleep 10 + yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + + # Test waltest + dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'` + walDir=`find $dataDir -name "wal"|head -n1` + echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT + echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT + if [ -n "$walDir" ]; then + yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir > /dev/null 2>&1 | tee -a $TDENGINE_COVERAGE_REPORT + fi + + # run Unit Test + echo "Run Unit Test: utilTest, queryTest and cliTest" + #$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null 2>&1 && echo "utilTest pass!" || echo "utilTest failed!" + #$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null 2>&1 && echo "queryTest pass!" || echo "queryTest failed!" + #$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null 2>&1 && echo "cliTest pass!" || echo "cliTest failed!" + + stopTaosd + + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + # collect data + lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + if [ "$branch" == "3.0" ]; then + lcov --remove coverage.info \ + '*/contrib/*' '*/tests/*' '*/test/*'\ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/tidpool.c' '*/tmempool.c'\ + '*/tthread.c' '*/tversion.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + else + lcov --remove coverage.info \ + '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' '*/ttype.h' '*/tarithoperator.c' '*/TSDBJNIConnector.c' '*/taosdemo.c'\ + --rc lcov_branch_coverage=1 -o coverage.info + fi + + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + + # push result to coveralls.io + echo "push result to coveralls.io" + /usr/local/bin/coveralls-lcov coverage.info -t o7uY02qEAgKyJHrkxLGiCOTfL3IGQR2zm | tee -a $TDENGINE_COVERAGE_REPORT + + #/root/pxiao/checkCoverageFile.sh -s $TDENGINE_DIR/source -f $TDENGINE_COVERAGE_REPORT + #cat /root/pxiao/fileListNoCoverage.log | tee -a $TDENGINE_COVERAGE_REPORT + cat $TDENGINE_COVERAGE_REPORT | grep "| 0.0%" | awk -F "%" '{print $1}' | awk -F "|" '{if($2==0.0)print $1}' | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function sendReport { + echo "send report" + receiver="develop@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT + BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT` + echo -e "from: \nto: ${receiver}\nsubject: Coverage test report ${branch} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \ + /usr/sbin/ssmtp "${receiver}" && echo "Report Sent!" +} + +function stopTaosd { + echo "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + echo "Stop tasod end" +} + +function runTestRandomFail { + exec_random_fail_sh=$1 + default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh + [ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1 + + dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py + default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py + [ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1 + + runTest NoRandomFail +} + +WORK_DIR=/root/pxiao + +date >> $WORK_DIR/cron.log +echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log + +stopTaosd +buildTDengine + +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh +#runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh + +runTest + +lcovFunc +#sendReport +stopTaosd + +date >> $WORK_DIR/cron.log +echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d8f4a36261..3e0e3ff7cf 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -187,9 +187,6 @@ # ---- snode ---- # unsupport ./test.sh -f tsim/snode/basic1.sim -# ---- bnode -./test.sh -f tsim/bnode/basic1.sim - # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim @@ -249,12 +246,18 @@ ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim -#./test.sh -f tsim/stream/partitionbyColumnInterval.sim +./test.sh -f tsim/stream/partitionbyColumnInterval.sim #./test.sh -f tsim/stream/partitionbyColumnSession.sim #./test.sh -f tsim/stream/partitionbyColumnState.sim #./test.sh -f tsim/stream/deleteInterval.sim #./test.sh -f tsim/stream/deleteSession.sim #./test.sh -f tsim/stream/deleteState.sim +#./test.sh -f tsim/stream/fillIntervalDelete0.sim +#./test.sh -f tsim/stream/fillIntervalDelete1.sim +./test.sh -f tsim/stream/fillIntervalLinear.sim +#./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +./test.sh -f tsim/stream/fillIntervalPrevNext.sim +./test.sh -f tsim/stream/fillIntervalValue.sim # ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim @@ -310,7 +313,7 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim -./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +# ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim # temp disable ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim diff --git a/tests/script/tmp/monitor.sim b/tests/script/tmp/monitor.sim index b410e1b6ad..1734b9a1a7 100644 --- a/tests/script/tmp/monitor.sim +++ b/tests/script/tmp/monitor.sim @@ -23,5 +23,4 @@ sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 bin print =============== create drop qnode 1 sql create qnode on dnode 1 #sql create snode on dnode 1 -#sql create bnode on dnode 1 diff --git a/tests/script/tsim/bnode/basic1.sim b/tests/script/tsim/bnode/basic1.sim deleted file mode 100644 index 0a20001636..0000000000 --- a/tests/script/tsim/bnode/basic1.sim +++ /dev/null @@ -1,135 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -sql connect - -print =============== select * from information_schema.ins_dnodes -sql select * from information_schema.ins_dnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -sql select * from information_schema.ins_mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -print =============== create dnodes -sql create dnode $hostname port 7200 -sleep 2000 - -sql select * from information_schema.ins_dnodes; -if $rows != 2 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data10 != 2 then - return -1 -endi - -print $data02 -if $data02 != 0 then - return -1 -endi - -if $data12 != 0 then - return -1 -endi - -if $data04 != ready then - return -1 -endi - -if $data14 != ready then - return -1 -endi - -sql select * from information_schema.ins_mnodes; -if $rows != 1 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data02 != leader then - return -1 -endi - -#print =============== create drop bnode 1 -#sql create bnode on dnode 1 -#sql show bnodes -#if $rows != 1 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#sql_error create bnode on dnode 1 -# -#sql drop bnode on dnode 1 -#sql show bnodes -#if $rows != 0 then -# return -1 -#endi -#sql_error drop bnode on dnode 1 -# -#print =============== create drop bnode 2 -#sql create bnode on dnode 2 -#sql show bnodes -#if $rows != 1 then -# return -1 -#endi -#if $data00 != 2 then -# return -1 -#endi -#sql_error create bnode on dnode 2 -# -#sql drop bnode on dnode 2 -#sql show bnodes -#if $rows != 0 then -# return -1 -#endi -#sql_error drop bnode on dnode 2 -# -#print =============== create drop bnodes -#sql create bnode on dnode 1 -#sql create bnode on dnode 2 -#sql show bnodes -#if $rows != 2 then -# return -1 -#endi - -#print =============== restart -#system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode2 -s stop -x SIGINT -#system sh/exec.sh -n dnode1 -s start -#system sh/exec.sh -n dnode2 -s start -# -#sleep 2000 -#sql show bnodes -#if $rows != 2 then -# return -1 -#endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index 40dcec4080..5cac026b57 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -17,9 +17,9 @@ sql create table $table1 (ts timestamp, b binary(20)) sql create table $table2 (ts timestamp, b binary(20)) sql insert into $table1 values(now, "table_name") -sql insert into $table1 values(now-1m, "tablexname") +sql insert into $table1 values(now-3m, "tablexname") sql insert into $table1 values(now-2m, "tablexxx") -sql insert into $table1 values(now-2m, "table") +sql insert into $table1 values(now-1m, "table") sql select b from $table1 if $rows != 4 then diff --git a/tests/script/tsim/stream/session0.sim b/tests/script/tsim/stream/session0.sim index 3e0af354d8..afae8ef5da 100644 --- a/tests/script/tsim/stream/session0.sim +++ b/tests/script/tsim/stream/session0.sim @@ -216,12 +216,12 @@ if $data02 != 3.274823935 then goto loop2 endi -if $data03 != 1.800000000 then +if $data03 != 1.500000000 then print ======$data03 return -1 endi -if $data04 != 3.350000000 then +if $data04 != 3.500000000 then print ======$data04 return -1 endi diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index ab173c5929..ee6eefde26 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -5,15 +5,15 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 -sql select * from information_schema.ins_databases +sql create database test vgroups 1; +sql select * from information_schema.ins_databases; if $rows != 3 then return -1 endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double,id int); diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 877a2877b9..60a50f9560 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -349,7 +349,7 @@ endi if $rows != 3 then print ====loop4=rows=$rows -# goto loop4 + goto loop4 endi # row 0 diff --git a/tests/script/tsim/stream/triggerSession0.sim b/tests/script/tsim/stream/triggerSession0.sim index 2ea689ef78..1bef439884 100644 --- a/tests/script/tsim/stream/triggerSession0.sim +++ b/tests/script/tsim/stream/triggerSession0.sim @@ -13,7 +13,7 @@ endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t2(ts timestamp, a int, b int , c int, d double); sql create stream streams2 trigger window_close into streamt2 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t2 session(ts, 10s); @@ -58,16 +58,11 @@ endi sql insert into t2 values(1648791233002,1,2,3,1.0); sleep 300 sql select * from streamt2; -if $rows != 1 then +if $rows != 0 then print ======$rows return -1 endi -if $data01 != 6 then - print ======$data01 - return -1 -endi - sql insert into t2 values(1648791253003,1,2,3,1.0); sleep 300 sql select * from streamt2; diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index 89a4babb0b..ad8d70b089 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -128,5 +128,4 @@ run tsim/sync/threeReplica1VgElectWihtInsert.sim run tsim/sma/tsmaCreateInsertQuery.sim run tsim/sma/rsmaCreateInsertQuery.sim run tsim/valgrind/basic.sim -run tsim/valgrind/checkError.sim -run tsim/bnode/basic1.sim \ No newline at end of file +run tsim/valgrind/checkError.sim \ No newline at end of file diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 86f95755d0..1614c3be55 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -110,7 +110,6 @@ sql_error show dnodes sql_error show snodes sql_error show qnodes sql_error show mnodes -sql_error show bnodes sql_error show db.vgroups sql_error show db.stables sql_error show db.tables diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index c713e9fd14..1d733191b7 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -125,7 +125,7 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', - 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] + 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode'] for elem in dnode_infos: if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: tdLog.exit(f"{elem} is null!") diff --git a/tests/system-test/1-insert/alter_database.py b/tests/system-test/1-insert/alter_database.py new file mode 100644 index 0000000000..d3a55ee0a6 --- /dev/null +++ b/tests/system-test/1-insert/alter_database.py @@ -0,0 +1,58 @@ +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.buffer_boundary = [3,4097,8193,12289,16384] + self.buffer_error = [self.buffer_boundary[0]-1,self.buffer_boundary[-1]+1,12289,96] + # pages_boundary >= 64 + self.pages_boundary = [64,128,512] + self.pages_error = [self.pages_boundary[0]-1] + def alter_buffer(self): + tdSql.execute('create database db') + for buffer in self.buffer_boundary: + tdSql.execute(f'alter database db buffer {buffer}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][8],buffer) + tdSql.execute('drop database db') + tdSql.execute('create database db vgroups 10') + for buffer in self.buffer_error: + tdSql.error(f'alter database db buffer {buffer}') + tdSql.execute('drop database db') + + def alter_pages(self): + tdSql.execute('create database db') + for pages in self.pages_boundary: + tdSql.execute(f'alter database db pages {pages}') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + tdSql.checkEqual(tdSql.queryResult[0][10],pages) + tdSql.execute('drop database db') + tdSql.execute('create database db') + tdSql.query('select * from information_schema.ins_databases where name = "db"') + self.pages_error.append(tdSql.queryResult[0][10]) + for pages in self.pages_error: + tdSql.error(f'alter database db pages {pages}') + tdSql.execute('drop database db') + + def run(self): + tdSql.error('create database db1 vgroups 10 buffer 12289') + self.alter_buffer() + self.alter_pages() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/create_wrong_topic.py b/tests/system-test/7-tmq/create_wrong_topic.py new file mode 100644 index 0000000000..d18cb0260e --- /dev/null +++ b/tests/system-test/7-tmq/create_wrong_topic.py @@ -0,0 +1,77 @@ +import taos +import sys +import time +import socket +import os +import threading +from util.common import * + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(),logSql) + self.setsql = TDSetSql() + self.rowNum = 10 + self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + } + self.error_topic = ['avg','count','spread','stddev','sum','hyperloglog'] + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + def wrong_topic(self): + tdSql.prepare() + tdSql.execute('use db') + stbname = f'db.{tdCom.getLongName(5, "letters")}' + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + tdSql.execute(f"create table {stbname}_tb1 using {stbname} tags({tag_values[0]})") + self.insert_data(self.column_dict,f'{stbname}_tb1',self.rowNum) + for column in self.column_dict.keys(): + for func in self.error_topic: + if func.lower() != 'count' and column.lower() != 'ts': + tdSql.error(f'create topic tpn as select {func}({column}) from {stbname}') + elif func.lower() == 'count' : + tdSql.error(f'create topic tpn as select {func}(*) from {stbname}') + for column in self.column_dict.keys(): + if column.lower() != 'ts': + tdSql.error(f'create topic tpn as select apercentile({column},50) from {stbname}') + tdSql.error(f'create topic tpn as select leastquares({column},1,1) from {stbname}_tb1') + tdSql.error(f'create topic tpn as select HISTOGRAM({column},user_input,[1,3,5,7],0) from {stbname}') + tdSql.error(f'create topic tpn as select percentile({column},1) from {stbname}_tb1') + pass + def run(self): + self.wrong_topic() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 63a59c0dc3..d253c34a53 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -41,7 +41,7 @@ class TDTestCase: ifcheckdata = 0 ifManualCommit = 1 groupId = 'group.id:cgrp1' - autoCommit = 'enable.auto.commit:false' + autoCommit = 'enable.auto.commit:true' autoCommitInterval = 'auto.commit.interval.ms:1000' autoOffset = 'auto.offset.reset:earliest' @@ -87,8 +87,9 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) - tdLog.info("After waiting for a period of time, drop one stable") - time.sleep(3) + tdLog.info("After waiting for a commit notify, drop one stable") + #time.sleep(3) + tmqCom.getStartCommitNotifyFromTmqsim() tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) tdLog.info("wait result from consumer, then check it") diff --git a/tests/system-test/7-tmq/tmqAutoCreateTbl.py b/tests/system-test/7-tmq/tmqAutoCreateTbl.py index a613f11267..ce2cad4b14 100644 --- a/tests/system-test/7-tmq/tmqAutoCreateTbl.py +++ b/tests/system-test/7-tmq/tmqAutoCreateTbl.py @@ -42,7 +42,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 3, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -52,7 +52,7 @@ class TDTestCase: paraDict['rowsPerTbl'] = self.rowsPerTbl tmqCom.initConsumerTable() - tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1, wal_retention_size=-1,wal_retention_period=-1) tdLog.info("create stb") tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) # tdLog.info("create ctb") @@ -87,7 +87,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 500, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -160,7 +160,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 1000, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index c0f33d9204..0691da6786 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -37,7 +37,7 @@ class TDTestCase: 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 10, + 'ctbNum': 100, 'rowsPerTbl': 4000, 'batchNum': 15, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 @@ -124,7 +124,8 @@ class TDTestCase: tdLog.info("async insert data") pThread = tmqCom.asyncInsertData(paraDict) - time.sleep(5) + tmqCom.getStartConsumeNotifyFromTmqsim(); + #time.sleep(5) tdLog.info("check show consumers") tdSql.query("show consumers") # tdLog.info(tdSql.queryResult) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index cc85b8ddc8..cc5836cbfd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -#python3 ./test.py -f 0-others/taosShell.py +python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 0-others/compatibility.py - +python3 ./test.py -f 1-insert/alter_database.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py @@ -232,7 +232,7 @@ python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDbRep3.py -N 5 -M 3 @@ -248,7 +248,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 5 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py -python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3 @@ -279,7 +279,7 @@ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_ python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 # python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py -N 4 -M 1 - +python3 ./test.py -f 7-tmq/create_wrong_topic.py python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index eedb6d7295..4d40de66bd 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -22,7 +22,7 @@ #define TAOS_CONSOLE_PROMPT_HEADER "taos> " #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "The auth string to use when connecting to the server." +#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 0b26d685fd..c34b22b691 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -101,11 +101,8 @@ void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { /* update the values */ cmd->commandSize += size; cmd->cursorOffset += size; - for (int i = 0; i < size; i++) { - taosMbToWchar(&wc, c + i, size); - cmd->screenOffset += taosWcharWidth(wc); - cmd->endOffset += taosWcharWidth(wc); - } + cmd->screenOffset += taosWcharWidth(wc); + cmd->endOffset += taosWcharWidth(wc); #ifdef WINDOWS #else shellShowOnScreen(cmd); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 28375d8a35..37d00c0a7c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -229,7 +229,7 @@ void shellRunSingleCommandImp(char *command) { printf("Query OK, %d of %d rows affected (%.6fs)\r\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); // call auto tab - callbackAutoTab(command, pSql, false); + callbackAutoTab(command, NULL, false); } printf("\r\n"); diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 2f1ee12d54..346757b76f 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -243,8 +243,8 @@ void enumAllWords(STireNode** nodes, char* prefix, SMatch* match) { continue; } else { // combine word string - memset(word, 0, sizeof(word)); - strcpy(word, prefix); + memset(word, 0, tListLen(word)); + strncpy(word, prefix, len); word[len] = FIRST_ASCII + i; // append current char // chain middle node @@ -275,7 +275,6 @@ void matchPrefixFromList(STire* tire, char* prefix, SMatch* match) { // match prefix words, if match is not NULL , put all item to match and return match void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { - SMatch* root = match; int m = 0; STireNode* c = 0; int len = strlen(prefix); @@ -299,24 +298,14 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) { // previous items already matched if (i == len - 1) { - // malloc match if not pass by param match - if (root == NULL) { - root = (SMatch*)taosMemoryMalloc(sizeof(SMatch)); - memset(root, 0, sizeof(SMatch)); - strcpy(root->pre, prefix); - } - // prefix is match to end char - if (c->d) enumAllWords(c->d, prefix, root); + if (c->d) enumAllWords(c->d, prefix, match); } else { // move to next node continue match if (c->d == NULL) break; nodes = c->d; } } - - // return - return; } SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) { diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000..1bdfca396c --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 1bdfca396cd6730cdc334e06fc7b2156dd1239a0 diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 56ba622a9c..ab520e6638 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -63,6 +63,7 @@ int smlProcess_influx_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); return code; } @@ -86,6 +87,8 @@ int smlProcess_telnet_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -99,32 +102,15 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[" - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 0," - " \"value\": 18," - " \"tags\": {" - " \"host\": \"web01\"," - " \"id\": \"t1\"," - " \"dc\": \"lga\"" - " }" - " }," - " {" - " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 1662344042," - " \"value\": 9," - " \"tags\": {" - " \"host\": \"web02\"," - " \"dc\": \"lga\"" - " }" - " }" - "]"}; + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -138,33 +124,15 @@ int smlProcess_json2_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current0\"," - " \"timestamp\": {" - " \"value\" : 1662344042," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"groupid\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"location\": { " - " \"value\" : \"北京\"," - " \"type\" : \"binary\"" - " }," - " \"id\": \"d1001\"" - " }" - "}"}; + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -178,61 +146,14 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current1\"," - " \"timestamp\": {" - " \"value\" : 1662344042," - " \"type\" : \"s\"" - " }," - " \"value\": {" - " \"value\" : 10.3," - " \"type\" : \"i64\"" - " }," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 2," - " \"type\" : \"bigint\"" - " }," - " \"t2\": { " - " \"value\" : 2," - " \"type\" : \"int\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"i16\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"i8\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"f32\"" - " }," - " \"t6\": { " - " \"value\" : 2," - " \"type\" : \"double\"" - " }," - " \"t7\": { " - " \"value\" : \"8323\"," - " \"type\" : \"binary\"" - " }," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": { " - " \"value\" : true," - " \"type\" : \"bool\"" - " }," - " \"id\": \"d1001\"" - " }" - "}"}; + "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -246,52 +167,15 @@ int smlProcess_json4_Test() { taos_free_result(pRes); const char *sql[] = { - "{" - " \"metric\": \"meter_current2\"," - " \"timestamp\": {" - " \"value\" : 1662344042000," - " \"type\" : \"ms\"" - " }," - " \"value\": \"ni\"," - " \"tags\": {" - " \"t1\": { " - " \"value\" : 20," - " \"type\" : \"i64\"" - " }," - " \"t2\": { " - " \"value\" : 25," - " \"type\" : \"i32\"" - " }," - " \"t3\": { " - " \"value\" : 2," - " \"type\" : \"smallint\"" - " }," - " \"t4\": { " - " \"value\" : 2," - " \"type\" : \"tinyint\"" - " }," - " \"t5\": { " - " \"value\" : 2," - " \"type\" : \"float\"" - " }," - " \"t6\": { " - " \"value\" : 0.2," - " \"type\" : \"f64\"" - " }," - " \"t7\": \"nsj\"," - " \"t8\": { " - " \"value\" : \"北京\"," - " \"type\" : \"nchar\"" - " }," - " \"t9\": false," - " \"id\": \"d1001\"" - " }" - "}"}; + "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" + }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -313,6 +197,8 @@ int sml_TD15662_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -333,6 +219,8 @@ int sml_TD15742_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -362,6 +250,8 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -781,6 +671,8 @@ int sml_oom_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -825,6 +717,8 @@ int sml_16368_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -862,6 +756,8 @@ int sml_dup_time_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -1068,6 +964,8 @@ int sml_16960_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); + return code; } @@ -1097,6 +995,7 @@ int sml_add_tag_col_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); code = taos_errno(pRes); taos_free_result(pRes); + taos_close(taos); return code; } @@ -1151,6 +1050,36 @@ int smlProcess_18784_Test() { rowIndex++; } taos_free_result(pRes); + taos_close(taos); + + return code; +} + +int sml_19221_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + const char *sql[] = { + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + }; + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + char* tmp = (char*)taosMemoryCalloc(1024, 1); + memcpy(tmp, sql[0], strlen(sql[0])); + *(char*)(tmp+44) = 0; + int32_t totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + + ASSERT(totalRows == 3); + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + taosMemoryFree(tmp); return code; } @@ -1187,5 +1116,7 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_18784_Test(); ASSERT(!ret); + ret = sml_19221_Test(); + ASSERT(!ret); return ret; } diff --git a/utils/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c index 784b45c92b..d105b50579 100644 --- a/utils/test/c/tmqDemo.c +++ b/utils/test/c/tmqDemo.c @@ -130,15 +130,15 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-s") == 0) { - strcpy(g_stConfInfo.stbName, argv[++i]); + tstrncpy(g_stConfInfo.stbName, argv[++i], sizeof(g_stConfInfo.stbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.vnodeWalPath, argv[++i]); + tstrncpy(g_stConfInfo.vnodeWalPath, argv[++i], sizeof(g_stConfInfo.vnodeWalPath)); } else if (strcmp(argv[i], "-f") == 0) { - strcpy(g_stConfInfo.resultFileName, argv[++i]); + tstrncpy(g_stConfInfo.resultFileName, argv[++i], sizeof(g_stConfInfo.resultFileName)); } else if (strcmp(argv[i], "-t") == 0) { g_stConfInfo.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index a043aa7a6d..25e83363ee 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -155,7 +155,7 @@ static void printHelp() { printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s%s%d\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); printf("%s%s\n", indent, "-b"); @@ -238,7 +238,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %d\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; @@ -263,11 +263,11 @@ void parseArgument(int32_t argc, char* argv[]) { printHelp(); exit(0); } else if (strcmp(argv[i], "-d") == 0) { - strcpy(g_stConfInfo.dbName, argv[++i]); + tstrncpy(g_stConfInfo.dbName, argv[++i], sizeof(g_stConfInfo.dbName)); } else if (strcmp(argv[i], "-w") == 0) { - strcpy(g_stConfInfo.cdbName, argv[++i]); + tstrncpy(g_stConfInfo.cdbName, argv[++i], sizeof(g_stConfInfo.cdbName)); } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); + tstrncpy(configDir, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { @@ -279,9 +279,9 @@ void parseArgument(int32_t argc, char* argv[]) { } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); } else if (strcmp(argv[i], "-t") == 0) { - char tmpBuf[56]; - strcpy(tmpBuf, argv[++i]); - sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + char tmpBuf[56] = {0}; + tstrncpy(tmpBuf, argv[++i], sizeof(tmpBuf)); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); } else if (strcmp(argv[i], "-x") == 0) { g_stConfInfo.numOfThread = atol(argv[++i]); } else if (strcmp(argv[i], "-l") == 0) { @@ -294,6 +294,10 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.producerRate = atol(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { g_stConfInfo.payloadLen = atol(argv[++i]); + if(g_stConfInfo.payloadLen <= 0 || g_stConfInfo.payloadLen > 1024 * 1024 * 1024){ + pError("%s calloc size is too large: %s %s", GREEN, argv[++i], NC); + exit(-1); + } } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -354,8 +358,8 @@ void ltrim(char* str) { int queryDB(TAOS* taos, char* command) { int retryCnt = 10; - int code; - TAOS_RES* pRes; + int code = 0; + TAOS_RES* pRes = NULL; while (retryCnt--) { pRes = taos_query(taos, command); @@ -363,10 +367,11 @@ int queryDB(TAOS* taos, char* command) { if (code != 0) { taosSsleep(1); taos_free_result(pRes); + pRes = NULL; continue; } taos_free_result(pRes); - return 0; + return 0; } pError("failed to reason:%s, sql: %s", tstrerror(code), command); @@ -418,7 +423,7 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { - taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", (int)strlen(buf)); taosCloseFile(&g_fp); return -1; } @@ -592,7 +597,7 @@ static int32_t data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -644,7 +649,7 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t vgroupId = tmq_get_vgroup_id(msg); const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%d\n", pInfo->consumerId, msgIndex); taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", tmq_get_topic_name(msg), vgroupId); @@ -944,7 +949,7 @@ void parseConsumeInfo() { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); - strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token); + tstrncpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token, sizeof(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic])); ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; @@ -960,7 +965,7 @@ void parseConsumeInfo() { ltrim(pstr); char* ret = strchr(pstr, ch); memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr); - strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1); + tstrncpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1, sizeof(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey])); // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey], // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; @@ -1268,25 +1273,26 @@ void* ombProduceThreadFunc(void* param) { for (int i = 0; i < batchPerTblTimes; ++i) { uint32_t msgsOfSql = g_stConfInfo.batchSize; if ((i == batchPerTblTimes - 1) && (0 != remainder)) { - msgsOfSql = remainder; + msgsOfSql = remainder; } int len = 0; len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); for (int j = 0; j < msgsOfSql; j++) { - int64_t timeStamp = taosGetTimestampNs(); - len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); sendMsgs++; pInfo->totalProduceMsgs++; } - totalMsgLen += len; + totalMsgLen += len; pInfo->totalMsgsLen += len; - int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { taos_close(pInfo->taos); - pInfo->taos = NULL; - return NULL; + pInfo->taos = NULL; + taosMemoryFree(sqlBuf); + return NULL; } affectedRowsTotal += affectedRows; @@ -1322,6 +1328,7 @@ void* ombProduceThreadFunc(void* param) { printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); taos_close(pInfo->taos); pInfo->taos = NULL; + taosMemoryFree(sqlBuf); return NULL; } diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 3a598ba98b..6540fdac4c 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -663,7 +663,7 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { if(strcmp(argv[i], "-c") == 0){ - strcpy(g_conf.dir, argv[++i]); + tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); }else if(strcmp(argv[i], "-s") == 0){ g_conf.snapShot = true; }else if(strcmp(argv[i], "-d") == 0){ diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 4fe5fdf672..6f693cf015 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -37,6 +37,7 @@ void simLogSql(char *sql, bool useSharp) { taosFsyncFile(pFile); } +#if 0 char *simParseArbitratorName(char *varName) { static char hostName[140]; #ifdef WINDOWS @@ -47,6 +48,7 @@ char *simParseArbitratorName(char *varName) { #endif return hostName; } +#endif char *simParseHostName(char *varName) { static char hostName[140]; @@ -102,9 +104,11 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { return simParseHostName(varName); } +#if 0 if (strncmp(varName, "arbitrator", 10) == 0) { return simParseArbitratorName(varName); } +#endif if (strncmp(varName, "error", varLen) == 0) return script->error; @@ -883,6 +887,7 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { return simExecuteSqlImpCmd(script, rest, isSlow); } +#if 0 bool simExecuteRestfulCmd(SScript *script, char *rest) { TdFilePtr pFile = NULL; char filename[256]; @@ -924,6 +929,7 @@ bool simExecuteRestfulCmd(SScript *script, char *rest) { return simExecuteSystemCmd(script, cmd); } +#endif bool simExecuteSqlErrorCmd(SScript *script, char *rest) { char buf[3000]; @@ -981,6 +987,7 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return false; } +#if 0 bool simExecuteLineInsertCmd(SScript *script, char *rest) { char buf[TSDB_MAX_BINARY_LEN] = {0}; @@ -1037,3 +1044,4 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { return true; } } +#endif \ No newline at end of file diff --git a/utils/tsim/src/simParse.c b/utils/tsim/src/simParse.c index 14be10bc8a..ae523edfc4 100644 --- a/utils/tsim/src/simParse.c +++ b/utils/tsim/src/simParse.c @@ -501,6 +501,7 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { char *token; int32_t tokenLen; @@ -647,6 +648,7 @@ bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { numOfLines++; return true; } +#endif bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -715,6 +717,7 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; @@ -726,6 +729,7 @@ bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; return true; } +#endif bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -838,6 +842,7 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +#if 0 bool simParseLineInsertCmd(char *rest, SCommand *pCmd, int32_t lineNum) { int32_t expLen; @@ -869,6 +874,7 @@ bool simParseLineInsertErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { numOfLines++; return true; } +#endif void simInitsimCmdList() { int32_t cmdno; @@ -930,6 +936,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_SWITCH; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "switch"); @@ -977,6 +984,7 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseEndsCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif cmdno = SIM_CMD_SLEEP; simCmdList[cmdno].cmdno = cmdno; @@ -1050,6 +1058,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_SQL_SLOW; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql_slow"); @@ -1065,6 +1074,7 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseRestfulCmd; simCmdList[cmdno].executeCmd = simExecuteRestfulCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif /* test is only an internal command */ cmdno = SIM_CMD_TEST; @@ -1082,6 +1092,7 @@ void simInitsimCmdList() { simCmdList[cmdno].executeCmd = simExecuteReturnCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#if 0 cmdno = SIM_CMD_LINE_INSERT; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "line_insert"); @@ -1097,4 +1108,5 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); +#endif }