diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 423169c007..ed12a0628b 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -38,40 +38,21 @@ def pre_test(){ sh ''' cd ${WK} git reset --hard + git remote prune origin + git fetch cd ${WKC} git reset --hard git clean -fxd + git remote prune origin + git fetch ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - cd ${WKC} - git checkout master - ''' - } else if(env.CHANGE_TARGET == '2.0') { - sh ''' - cd ${WK} - git checkout 2.0 - cd ${WKC} - git checkout 2.0 - ''' - } else if(env.CHANGE_TARGET == '3.0') { - sh ''' - cd ${WK} - git checkout 3.0 - cd ${WKC} - git checkout 3.0 - ''' - } else { - sh ''' - cd ${WK} - git checkout develop - cd ${WKC} - git checkout develop - ''' - } + sh ''' + cd ${WK} + git checkout ''' + env.CHANGE_TARGET + ''' + cd ${WKC} + git checkout ''' + env.CHANGE_TARGET + ''' + ''' } if (env.CHANGE_URL =~ /\/TDengine\//) { sh ''' @@ -169,49 +150,24 @@ def pre_test_win(){ bat ''' cd %WIN_INTERNAL_ROOT% git reset --hard + git remote prune origin + git fetch ''' bat ''' cd %WIN_COMMUNITY_ROOT% git reset --hard + git remote prune origin + git fetch ''' script { - if (env.CHANGE_TARGET == 'master') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout master - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout master - ''' - } else if(env.CHANGE_TARGET == '2.0') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout 2.0 - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout 2.0 - ''' - } else if(env.CHANGE_TARGET == '3.0') { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout 3.0 - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout 3.0 - ''' - } else { - bat ''' - cd %WIN_INTERNAL_ROOT% - git checkout develop - ''' - bat ''' - cd %WIN_COMMUNITY_ROOT% - git checkout develop - ''' - } + bat ''' + cd %WIN_INTERNAL_ROOT% + git checkout ''' + env.CHANGE_TARGET + ''' + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git checkout ''' + env.CHANGE_TARGET + ''' + ''' } script { if (env.CHANGE_URL =~ /\/TDengine\//) { diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ee1e7d3b31..ed8216be91 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ed6a160 + GIT_TAG 3d21433 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 4f08656ac8..ad09ce24cf 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2d68404 + GIT_TAG 57bdfbf SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index c6d42b686c..506559a245 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 97c4bac + GIT_TAG 7a54d21 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 002b515093..c947e86d1c 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -47,27 +47,28 @@ If the displayed content is followed by `...` you can use this command to change You can change the behavior of TDengine CLI by specifying command-line parameters. The following parameters are commonly used. -- -h, --host=HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service -- -P, --port=PORT: Specify the port number to be used by the server. Default is `6030` -- -u, --user=USER: the user name to use when connecting. Default is `root` -- -p, --password=PASSWORD: the password to use when connecting to the server. Default is `taosdata` +- -h HOST: FQDN of the server where the TDengine server is to be connected. Default is to connect to the local service +- -P PORT: Specify the port number to be used by the server. Default is `6030` +- -u USER: the user name to use when connecting. Default is `root` +- -p PASSWORD: the password to use when connecting to the server. Default is `taosdata` - -?, --help: print out all command-line arguments And many more parameters. -- -c, --config-dir: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` -- -C, --dump-config: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c -- -d, --database=DATABASE: Specify the database to use when connecting to the server -- -D, --directory=DIRECTORY: Import the SQL script file in the specified path -- -f, --file=FILE: Execute the SQL script file in non-interactive mode -- -k, --check=CHECK: Specify the table to be checked -- -l, --pktlen=PKTLEN: Test package size to be used for network testing -- -n, --netrole=NETROLE: test scope for network connection test, default is `startup`. The value can be `client`, `server`, `rpc`, `startup`, `sync`, `speed`, or `fqdn`. -- -r, --raw-time: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) -- -s, --commands=COMMAND: execute SQL commands in non-interactive mode -- -S, --pkttype=PKTTYPE: Specify the packet type used for network testing. The default is TCP, can be specified as either TCP or UDP when `speed` is specified to `netrole` parameter -- -T, --thread=THREADNUM: The number of threads to import data in multi-threaded mode -- -s, --commands: Run TDengine CLI commands without entering the terminal +- -a AUTHSTR: The auth string to use when connecting to the server +- -A: Generate auth string from password +- -c CONFIGDIR: Specify the directory where configuration file exists. The default is `/etc/taos`, and the default name of the configuration file in this directory is `taos.cfg` +- -C: Print the configuration parameters of `taos.cfg` in the default directory or specified by -c +- -d DATABASE: Specify the database to use when connecting to the server +- -f FILE: Execute the SQL script file in non-interactive mode +- -k: Check the service status, 0: unavailable,1: network ok,2: service ok,3: service degraded,4: exiting +- -l PKTLEN: Test package length to be used for network testing +- -n NETROLE: test scope for network connection test, default is `client`. The value can be `client`, `server` +- -N PKTNUM: Test package numbers to be used for network testing +- -r: output the timestamp format as unsigned 64-bits integer (uint64_t in C language) +- -s COMMAND: execute SQL commands in non-interactive mode +- -t: Check the details of the service status,status same as -k +- -w DISPLAYWIDTH: 客户端列显示宽度 - -z, --timezone=TIMEZONE: Specify time zone. Default is the value of current configuration file - -V, --version: Print out the current version number diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index af15758225..656344eff4 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -5,12 +5,11 @@ description: "List of platforms supported by TDengine server, client, and connec ## List of supported platforms for TDengine server -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | -| ------------ | -------------- | ------------------- | --------------- | -| X64 | ● | ● | | -| MIPS64 | | | ● | -| ARM64 | | ○ | ○ | -| Alpha64 | | | ○ | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **UOS** | **Kylin** | **Ningsi V60/V80** | **HUAWEI EulerOS** | +| ------------------ | ----------------- | ---------------- | ---------------- | --------------- | ------- | --------- | ------------------ | ------------------ | +| X64 | ● | ● | ● | | ● | ● | ● | | +| Raspberry Pi ARM64 | | | | ● | | | | | +| HUAWEI cloud ARM64 | | | | | | | | ● | Note: ● means officially tested and verified, ○ means unofficially tested and verified. diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index d6cffd22e0..118bce8037 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -25,7 +25,6 @@ All executable files of TDengine are in the _/usr/local/taos/bin_ directory by d - _taosBenchmark_: TDengine testing tool - _remove.sh_: script to uninstall TDengine, please execute it carefully, link to the **rmtaos** command in the /usr/bin directory. Will remove the TDengine installation directory `/usr/local/taos`, but will keep `/etc/taos`, `/var/lib/taos`, `/var/log/taos` - _taosadapter_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares -- _tarbitrator_: provides arbitration for two-node cluster deployments - _TDinsight.sh_: script to download TDinsight and install it - _set_core.sh_: script for setting up the system to generate core dump files for easy debugging - _taosd-dump-cfg.gdb_: script to facilitate debugging of taosd's gdb execution. diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index 741f6dfeab..8b8193a99b 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -2,6 +2,9 @@ sidebar_label: Docker title: 通过 Docker 快速体验 TDengine --- +:::info +如果您希望对 TDengine 贡献代码或对内部实现感兴趣,请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +::: 本节首先介绍如何通过 Docker 快速体验 TDengine,然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。 @@ -63,6 +66,10 @@ taos> taosAdapter 是 TDengine 中提供 REST 服务的组件。下面这条命令会在容器中同时启动 `taosd` 和 `taosadapter` 两个服务组件。默认 Docker 镜像同时启动 TDengine 后台服务 taosd 和 taosAdatper。 +```shell +docker run -d --name tdengine -p 6041:6041 tdengine/tdengine +``` + 可以在宿主机使用 curl 通过 RESTful 端口访问 Docker 容器内的 TDengine server。 ``` diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 7ff7da2bfb..024742ed8b 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -89,10 +89,6 @@ T = 最新事件时间 - watermark 无论在哪种模式下,watermark 都应该被妥善设置,来得到正确结果(直接丢弃模式)或避免频繁触发重算带来的性能开销(重新计算模式)。 -## 流式计算的数据填充策略 - -TODO - ## 流式计算与会话窗口(session window) ```sql @@ -105,14 +101,6 @@ window_clause: { 其中,SESSION 是会话窗口,tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口,如果连续的两条数据的时间超过 tol_val,则自动开启下一个窗口。 -## 流式计算的监控与流任务分布查询 - -TODO - -## 流式计算的内存控制与存算分离 - -TODO - ## 流式计算的暂停与恢复 ```sql diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index 1e17870685..8139b2fc55 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -1,6 +1,6 @@ --- -sidebar_label: 元数据库 -title: 元数据库 +sidebar_label: 元数据 +title: 存储元数据的 Information_Schema 数据库 --- TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库,提供对数据库元数据、数据库系统信息和状态的访问,例如数据库或表的名称,当前执行的 SQL 语句等。该数据库存储有关 TDengine 维护的所有其他数据库的信息。它包含多个只读表。实际上,这些表都是视图,而不是基表,因此没有与它们关联的文件。所以对这些表只能查询,不能进行 INSERT 等写入操作。`INFORMATION_SCHEMA` 数据库旨在以一种更一致的方式来提供对 TDengine 支持的各种 SHOW 语句(如 SHOW TABLES、SHOW DATABASES)所提供的信息的访问。与 SHOW 语句相比,使用 SELECT ... FROM INFORMATION_SCHEMA.tablename 具有以下优点: diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md new file mode 100644 index 0000000000..1ff7da6c71 --- /dev/null +++ b/docs/zh/12-taos-sql/23-perf.md @@ -0,0 +1,129 @@ +--- +sidebar_label: 统计数据 +title: 存储统计数据的 Performance_Schema 数据库 +--- + +TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`,其中存储了与性能有关的统计数据。本节详细介绍其中的表和表结构。 + +## PERF_APP + +提供接入集群的应用(客户端)的相关信息。也可以使用 SHOW APPS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ------------------------------- | +| 1 | app_id | UBIGINT | 客户端 ID | +| 2 | ip | BINARY(16) | 客户端地址 | +| 3 | pid | INT | 客户端进程 号 | +| 4 | name | BINARY(24) | 客户端名称 | +| 5 | start_time | TIMESTAMP | 客户端启动时间 | +| 6 | insert_req | UBIGINT | insert 请求次数 | +| 7 | insert_row | UBIGINT | insert 插入行数 | +| 8 | insert_time | UBIGINT | insert 请求的处理时间,单位微秒 | +| 9 | insert_bytes | UBIGINT | insert 请求消息字节数 | +| 10 | fetch_bytes | UBIGINT | 查询结果字节数 | +| 11 | query_time | UBIGINT | 查询请求处理时间 | +| 12 | slow_query | UBIGINT | 慢查询(处理时间 >= 3 秒)个数 | +| 13 | total_req | UBIGINT | 总请求数 | +| 14 | current_req | UBIGINT | 当前正在处理的请求个数 | +| 15 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_CONNECTIONS + +数据库的连接的相关信息。也可以使用 SHOW CONNECTIONS 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | -------------------------------------------------- | +| 1 | conn_id | INT | 连接 ID | +| 2 | user | BINARY(24) | 用户名 | +| 3 | app | BINARY(24) | 客户端名称 | +| 4 | pid | UINT | 发起此连接的客户端在自己所在服务器或主机上的进程号 | +| 5 | end_point | BINARY(128) | 客户端地址 | +| 6 | login_time | TIMESTAMP | 登录时间 | +| 7 | last_access | TIMESTAMP | 最后更新时间 | + +## PERF_QUERIES + +提供当前正在执行的 SQL 语句的信息。也可以使用 SHOW QUERIES 来查询这些信息。 + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | ---------------------------- | +| 1 | kill_id | UBIGINT | 用来停止查询的 ID | +| 2 | query_id | INT | 查询 ID | +| 3 | conn_id | UINT | 连接 ID | +| 4 | app | BINARY(24) | app 名称 | +| 5 | pid | INT | app 在自己所在主机上的进程号 | +| 6 | user | BINARY(24) | 用户名 | +| 7 | end_point | BINARY(16) | 客户端地址 | +| 8 | create_time | TIMESTAMP | 创建时间 | +| 9 | exec_usec | BIGINT | 已执行时间 | +| 10 | stable_query | BOOL | 是否是超级表查询 | +| 11 | sub_num | INT | 子查询数量 | +| 12 | sub_status | BINARY(1000) | 子查询状态 | +| 13 | sql | BINARY(1024) | SQL 语句 | + +## PERF_TOPICS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------ | +| 1 | topic_name | BINARY(192) | topic 名称 | +| 2 | db_name | BINARY(64) | topic 相关的 DB | +| 3 | create_time | TIMESTAMP | topic 的 创建时间 | +| 4 | sql | BINARY(1024) | 创建该 topic 时所用的 SQL 语句 | + +## PERF_CONSUMERS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ----------------------------------------------------------- | +| 1 | consumer_id | BIGINT | 消费者的唯一 ID | +| 2 | consumer_group | BINARY(192) | 消费者组 | +| 3 | client_id | BINARY(192) | 用户自定义字符串,通过创建 consumer 时指定 client_id 来展示 | +| 4 | status | BINARY(20) | 消费者当前状态 | +| 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic,则展示为多行 | +| 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | +| 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | +| 8 | rebalance_time | TIMESTAMP | 上一次触发 rebalance 的时间 | + +## PERF_SUBSCRIPTIONS + +| # | **列名** | **数据类型** | **说明** | +| --- | :------------: | ------------ | ------------------------ | +| 1 | topic_name | BINARY(204) | 被订阅的 topic | +| 2 | consumer_group | BINARY(193) | 订阅者的消费者组 | +| 3 | vgroup_id | INT | 消费者被分配的 vgroup id | +| 4 | consumer_id | BIGINT | 消费者的唯一 id | + +## PERF_TRANS + +| # | **列名** | **数据类型** | **说明** | +| --- | :--------------: | ------------ | -------- | +| 1 | id | INT | | +| 2 | create_time | TIMESTAMP | | +| 3 | stage | BINARY(12) | | +| 4 | db1 | BINARY(64) | | +| 5 | db2 | BINARY(64) | | +| 6 | failed_times | INT | | +| 7 | last_exec_time | TIMESTAMP | | +| 8 | last_action_info | BINARY(511) | | + +## PERF_SMAS + +| # | **列名** | **数据类型** | **说明** | +| --- | :---------: | ------------ | ------------------------------------------- | +| 1 | sma_name | BINARY(192) | 时间维度的预计算 (time-range-wise sma) 名称 | +| 2 | create_time | TIMESTAMP | sma 创建时间 | +| 3 | stable_name | BINARY(192) | sma 所属的超级表名称 | +| 4 | vgroup_id | INT | sma 专属的 vgroup 名称 | + +## PERF_STREAMS + +| # | **列名** | **数据类型** | **说明** | +| --- | :----------: | ------------ | --------------------------------------- | +| 1 | stream_name | BINARY(64) | 流计算名称 | +| 2 | create_time | TIMESTAMP | 创建时间 | +| 3 | sql | BINARY(1024) | 创建流计算时提供的 SQL 语句 | +| 4 | status | BIANRY(20) | 流当前状态 | +| 5 | source_db | BINARY(64) | 源数据库 | +| 6 | target_db | BIANRY(64) | 目的数据库 | +| 7 | target_table | BINARY(192) | 流计算写入的目标表 | +| 8 | watermark | BIGINT | watermark,详见 SQL 手册流式计算 | +| 9 | trigger | INT | 计算结果推送模式,详见 SQL 手册流式计算 | diff --git a/docs/zh/14-reference/03-connector/03-connector.mdx b/docs/zh/14-reference/03-connector/03-connector.mdx index 7a4a85276e..00b76af867 100644 --- a/docs/zh/14-reference/03-connector/03-connector.mdx +++ b/docs/zh/14-reference/03-connector/03-connector.mdx @@ -10,14 +10,14 @@ TDengine 提供了丰富的应用程序开发接口,为了便于用户快速 目前 TDengine 的原生接口连接器可支持的平台包括:X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台,以及 Linux/Win64/Win32 等开发环境。对照矩阵如下: -| **CPU** | **OS** | **JDBC** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | +| **CPU** | **OS** | **Java** | **Python** | **Go** | **Node.js** | **C#** | **Rust** | C/C++ | | -------------- | --------- | -------- | ---------- | ------ | ----------- | ------ | -------- | ----- | | **X86 64bit** | **Linux** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win64** | ● | ● | ● | ● | ● | ● | ● | | **X86 64bit** | **Win32** | ● | ● | ● | ● | ○ | ○ | ● | | **X86 32bit** | **Win32** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **ARM64** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | -| **ARM32** | **Linux** | ● | ● | ● | ● | ○ | ○ | ● | +| **ARM32** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ● | | **MIPS 龙芯** | **Linux** | ○ | ○ | ○ | ○ | ○ | ○ | ○ | | **Alpha 申威** | **Linux** | ○ | ○ | -- | -- | -- | -- | ○ | | **X86 海光** | **Linux** | ○ | ○ | ○ | -- | -- | -- | ○ | @@ -32,6 +32,7 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器 | **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | | --------------------- | -------- | ---------- | ------------ | ------------- | --------------- | -------- | +| **3.0.0.0 及以上** | 3.0.0 | 当前版本 | 3.0 分支 | 3.0.0 | 3.0.0 | 当前版本 | | **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.6 及以上** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | | **2.4.0.4 - 2.4.0.5** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | @@ -48,9 +49,8 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器 | -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **连续查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **订阅功能** | 支持 | 支持 | 支持 | 支持 | 支持 | 暂不支持 | +| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | @@ -58,17 +58,17 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器 由于不同编程语言数据库框架规范不同,并不意味着所有 C/C++ 接口都需要对应封装支持。 ::: -### 使用 REST 接口 +### 使用 http (REST 或 WebSocket) 接口 | **功能特性** | **Java** | **Python** | **Go** | **C#(暂不支持)** | **Node.js** | **Rust** | | ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | | **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **参数绑定** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **订阅功能** | 不支持 | 不支持 | 不支持 | N/A | 不支持 | 不支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | -| **批量拉取(基于 WebSocket)** | 支持 | 暂不支持 | 暂不支持 | N/A | 暂不支持 | 暂不支持 | +| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | +| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 | +| **批量拉取(基于 WebSocket)** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 | :::warning diff --git a/docs/zh/14-reference/08-taos-shell.md b/docs/zh/14-reference/08-taos-shell.md index d4a516d2d2..2f3b551502 100644 --- a/docs/zh/14-reference/08-taos-shell.md +++ b/docs/zh/14-reference/08-taos-shell.md @@ -48,29 +48,30 @@ taos> SET MAX_BINARY_DISPLAY_WIDTH ; 您可通过配置命令行参数来改变 TDengine CLI 的行为。以下为常用的几个命令行参数: -- -h, --host=HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 -- -P, --port=PORT: 指定服务端所用端口号 -- -u, --user=USER: 连接时使用的用户名 -- -p, --password=PASSWORD: 连接服务端时使用的密码 +- -h HOST: 要连接的 TDengine 服务端所在服务器的 FQDN, 默认为连接本地服务 +- -P PORT: 指定服务端所用端口号 +- -u USER: 连接时使用的用户名 +- -p PASSWORD: 连接服务端时使用的密码 - -?, --help: 打印出所有命令行参数 还有更多其他参数: -- -c, --config-dir: 指定配置文件目录,Linux 环境下默认为 `/etc/taos`,该目录下的配置文件默认名称为 `taos.cfg` -- -C, --dump-config: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 -- -d, --database=DATABASE: 指定连接到服务端时使用的数据库 -- -D, --directory=DIRECTORY: 导入指定路径中的 SQL 脚本文件 -- -f, --file=FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 -- -k, --check=CHECK: 指定要检查的表 -- -l, --pktlen=PKTLEN: 网络测试时使用的测试包大小 -- -n, --netrole=NETROLE: 网络连接测试时的测试范围,默认为 `startup`, 可选值为 `client`、`server`、`rpc`、`startup`、`sync`、`speed` 和 `fqdn` 之一 -- -r, --raw-time: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) -- -s, --commands=COMMAND: 以非交互模式执行的 SQL 命令 -- -S, --pkttype=PKTTYPE: 指定网络测试所用的包类型,默认为 TCP。只有 netrole 为 `speed` 时既可以指定为 TCP 也可以指定为 UDP -- -T, --thread=THREADNUM: 以多线程模式导入数据时的线程数 -- -s, --commands: 在不进入终端的情况下运行 TDengine 命令 -- -z, --timezone=TIMEZONE: 指定时区,默认为本地时区 -- -V, --version: 打印出当前版本号 +- -a AUTHSTR: 连接服务端的授权信息 +- -A: 通过用户名和密码计算授权信息 +- -c CONFIGDIR: 指定配置文件目录,Linux 环境下默认为 `/etc/taos`,该目录下的配置文件默认名称为 `taos.cfg` +- -C: 打印 -c 指定的目录中 `taos.cfg` 的配置参数 +- -d DATABASE: 指定连接到服务端时使用的数据库 +- -f FILE: 以非交互模式执行 SQL 脚本文件。文件中一个 SQL 语句只能占一行 +- -k: 测试服务端运行状态,0: unavailable,1: network ok,2: service ok,3: service degraded,4: exiting +- -l PKTLEN: 网络测试时使用的测试包大小 +- -n NETROLE: 网络连接测试时的测试范围,默认为 `client`, 可选值为 `client`、`server` +- -N PKTNUM: 网络测试时使用的测试包数量 +- -r: 将时间输出出无符号 64 位整数类型(即 C 语音中 uint64_t) +- -s COMMAND: 以非交互模式执行的 SQL 命令 +- -t: 测试服务端启动状态,状态同-k +- -w DISPLAYWIDTH: 客户端列显示宽度 +- -z TIMEZONE: 指定时区,默认为本地时区 +- -V: 打印出当前版本号 示例: diff --git a/docs/zh/14-reference/09-support-platform/index.md b/docs/zh/14-reference/09-support-platform/index.md index d396e4be1f..e676ab845a 100644 --- a/docs/zh/14-reference/09-support-platform/index.md +++ b/docs/zh/14-reference/09-support-platform/index.md @@ -5,18 +5,11 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表" ## TDengine 服务端支持的平台列表 -| | **CentOS 7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | -| ------------ | -------------- | ------------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | -| X64 | ● | ● | | ○ | ● | ● | ● | -| 龙芯 MIPS64 | | | ● | | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | | -| 申威 Alpha64 | | | ○ | ● | | | | -| 飞腾 ARM64 | | ○ 优麒麟 | | | | | | -| 海光 X64 | ● | ● | ● | ○ | ● | ● | | -| 瑞芯微 ARM64 | | | ○ | | | | | -| 全志 ARM64 | | | ○ | | | | | -| 炬力 ARM64 | | | ○ | | | | | -| 华为云 ARM64 | | | | | | | ● | +| | **Windows 10/11** | **CentOS 7.9/8** | **Ubuntu 18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | +| ------------ | ----------------- | ---------------- | ---------------- | --------------- | ------------ | ----------------- | ---------------- | ---------------- | +| X64 | ● | ● | ● | | ● | ● | ● | | +| 树莓派 ARM64 | | | | ● | | | | | +| 华为云 ARM64 | | | | | | | | ● | 注: ● 表示经过官方测试验证, ○ 表示非官方测试验证。 diff --git a/docs/zh/14-reference/12-directory.md b/docs/zh/14-reference/12-directory.md index 0caf7e03c3..262eb99fa5 100644 --- a/docs/zh/14-reference/12-directory.md +++ b/docs/zh/14-reference/12-directory.md @@ -25,7 +25,6 @@ TDengine 的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 - _taosBenchmark_:TDengine 测试工具 - _remove.sh_:卸载 TDengine 的脚本,请谨慎执行,链接到/usr/bin 目录下的**rmtaos**命令。会删除 TDengine 的安装目录/usr/local/taos,但会保留/etc/taos、/var/lib/taos、/var/log/taos - _taosadapter_: 提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 -- _tarbitrator_: 提供双节点集群部署的仲裁功能 - _TDinsight.sh_:用于下载 TDinsight 并安装的脚本 - _set_core.sh_:用于方便调试设置系统生成 core dump 文件的脚本 - _taosd-dump-cfg.gdb_:用于方便调试 taosd 的 gdb 执行脚本。 diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1cdd4c02da..fc34915fe7 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -21,17 +21,17 @@ #include "taos.h" static int running = 1; -static char dbName[64] = "tmqdb"; -static char stbName[64] = "stb"; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; static char topicName[64] = "topicname"; static int32_t msg_process(TAOS_RES* msg) { - char buf[1024]; + char buf[1024]; int32_t rows = 0; const char* topicName = tmq_get_topic_name(msg); - const char* dbName = tmq_get_db_name(msg); - int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); printf("topic: %s\n", topicName); printf("db: %s\n", dbName); @@ -41,14 +41,14 @@ static int32_t msg_process(TAOS_RES* msg) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); + TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); - int32_t precision = taos_result_precision(msg); - const char* tbName = tmq_get_table_name(msg); - rows++; + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + rows++; taos_print_row(buf, row, fields, numOfFields); - printf("row content from %s: %s\n", (tbName != NULL ? tbName : "null table"), buf); + printf("row content from %s: %s\n", (tbName != NULL ? tbName : "table null"), buf); } return rows; @@ -80,7 +80,8 @@ static int32_t init_env() { // create super table printf("create super table\n"); - pRes = taos_query(pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + pRes = taos_query( + pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); if (taos_errno(pRes) != 0) { printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); return -1; @@ -166,7 +167,6 @@ int32_t create_topic() { } taos_free_result(pRes); - // pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3 from tmqdb.stb where c1 > 1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); @@ -184,26 +184,28 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; - tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "group.id", "cgrpName"); if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "client.id", "user defined name"); + if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.user", "root"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); if (TMQ_CONF_OK != code) return NULL; - code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "experimental.snapshot.enable", "true"); if (TMQ_CONF_OK != code) return NULL; code = tmq_conf_set(conf, "msg.with.table.name", "true"); if (TMQ_CONF_OK != code) return NULL; - - tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); - + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_conf_destroy(conf); return tmq; @@ -211,7 +213,7 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, "topicname"); if (code) { return NULL; } @@ -228,18 +230,18 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topicList) { int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t consumeDelay = 5000; + int32_t timeout = 5000; while (running) { - TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, consumeDelay); + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeout); if (tmqmsg) { msgCnt++; totalRows += msg_process(tmqmsg); taos_free_result(tmqmsg); - } else { - break; - } + /*} else {*/ + /*break;*/ + } } - + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } @@ -256,32 +258,30 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); if (NULL == tmq) { - fprintf(stderr, "%% build_consumer() fail!\n"); + fprintf(stderr, "%% build_consumer() fail!\n"); return -1; } tmq_list_t* topic_list = build_topic_list(); if (NULL == topic_list) { return -1; - } - + } + basic_consume_loop(tmq, topic_list); code = tmq_unsubscribe(tmq); if (code) { fprintf(stderr, "%% Failed to unsubscribe: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% unsubscribe\n"); } code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); - } - else { + } else { fprintf(stderr, "%% Consumer closed\n"); } - + return 0; } diff --git a/examples/rust b/examples/rust deleted file mode 160000 index 7ed7a97715..0000000000 --- a/examples/rust +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed7a97715388fa144718764d6bf20f9bfc29a12 diff --git a/include/client/taos.h b/include/client/taos.h index b7df0e4d29..dd7266bd96 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -196,15 +196,6 @@ DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, vo DLL_EXPORT void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res); -// Shuduo: temporary enable for app build -#if 1 -typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp, - void *param, int interval); -DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); -DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); -#endif - DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); @@ -281,10 +272,6 @@ DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); /* ------------------------------ TMQ END -------------------------------- */ -#if 1 // Shuduo: temporary enable for app build -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); -#endif - typedef enum { TSDB_SRV_STATUS_UNAVAILABLE = 0, TSDB_SRV_STATUS_NETWORK_OK = 1, diff --git a/include/common/tcommon.h b/include/common/tcommon.h index be18ef1fc0..e04d9d5e86 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -103,12 +103,12 @@ typedef struct SDataBlockInfo { int16_t hasVarCol; uint32_t capacity; // TODO: optimize and remove following - int64_t version; // used for stream, and need serialization - int64_t ts; // used for stream, and need serialization - int32_t childId; // used for stream, do not serialize - EStreamType type; // used for stream, do not serialize - STimeWindow calWin; // used for stream, do not serialize - TSKEY watermark;// used for stream + int64_t version; // used for stream, and need serialization + int64_t ts; // used for stream, and need serialization + int32_t childId; // used for stream, do not serialize + EStreamType type; // used for stream, do not serialize + STimeWindow calWin; // used for stream, do not serialize + TSKEY watermark; // used for stream } SDataBlockInfo; typedef struct SSDataBlock { @@ -268,6 +268,15 @@ typedef struct SSortExecInfo { int32_t readBytes; // read io bytes } SSortExecInfo; +// stream special block column + +#define START_TS_COLUMN_INDEX 0 +#define END_TS_COLUMN_INDEX 1 +#define UID_COLUMN_INDEX 2 +#define GROUPID_COLUMN_INDEX 3 +#define CALCULATE_START_TS_COLUMN_INDEX 4 +#define CALCULATE_END_TS_COLUMN_INDEX 5 + #ifdef __cplusplus } #endif diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 311eb72b4d..3679b3773b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -200,8 +200,6 @@ struct STag { #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP -#define TD_SUPPORT_READ2 -#define TD_SUPPORT_BACK2 // suppport back compatibility of 2.0 #define TASSERT(x) ASSERT(x) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9724386a37..df127af256 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1369,8 +1369,8 @@ typedef struct { int32_t numOfCols; int64_t skey; int64_t ekey; - int64_t version; // for stream - TSKEY watermark;// for stream + int64_t version; // for stream + TSKEY watermark; // for stream char data[]; } SRetrieveTableRsp; @@ -3080,6 +3080,22 @@ typedef struct SDeleteRes { int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); +typedef struct { + int64_t uid; + int64_t ts; +} SSingleDeleteReq; + +int32_t tEncodeSSingleDeleteReq(SEncoder* pCoder, const SSingleDeleteReq* pReq); +int32_t tDecodeSSingleDeleteReq(SDecoder* pCoder, SSingleDeleteReq* pReq); + +typedef struct { + int64_t suid; + SArray* deleteReqs; // SArray +} SBatchDeleteReq; + +int32_t tEncodeSBatchDeleteReq(SEncoder* pCoder, const SBatchDeleteReq* pReq); +int32_t tDecodeSBatchDeleteReq(SDecoder* pCoder, SBatchDeleteReq* pReq); + typedef struct { int32_t msgIdx; int32_t msgType; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 9ddb872007..6462c7afbf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -202,6 +202,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) TD_DEF_MSG_TYPE(TDMT_VND_FETCH_RSMA, "vnode-fetch-rsma", SRSmaFetchMsg, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL) diff --git a/include/common/trow.h b/include/common/trow.h index bd8510dde6..6cc9355144 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -319,21 +319,17 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); -void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal); + int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow); bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); -bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, - SCellVal *pVal); -bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); -void tdSCellValPrint(SCellVal *pVal, int8_t colType); void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_ROW_H_*/ +#endif /*_TD_COMMON_ROW_H_*/ \ No newline at end of file diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 72732ee198..e708a2c42d 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -54,10 +54,6 @@ typedef struct SFuncExecFuncs { FExecCombine combine; } SFuncExecFuncs; -typedef struct SFileBlockInfo { - int32_t numBlocksOfStep; -} SFileBlockInfo; - #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 @@ -171,8 +167,6 @@ typedef struct tExprNode { }; } tExprNode; -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); - struct SScalarParam { bool colAlloced; SColumnInfoData *columnData; @@ -182,14 +176,10 @@ struct SScalarParam { int32_t numOfRows; }; -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable); - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); typedef struct SPoint { int64_t key; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9355d76dcb..f51c37ed47 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -34,6 +34,8 @@ typedef struct SStreamTask SStreamTask; enum { STREAM_STATUS__NORMAL = 0, + STREAM_STATUS__STOP, + STREAM_STATUS__FAILED, STREAM_STATUS__RECOVER, }; @@ -118,7 +120,7 @@ typedef struct { int64_t sourceVer; int64_t reqId; - SArray* blocks; // SArray + SArray* blocks; // SArray } SStreamDataBlock; typedef struct { diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 35e79cf45d..aa563343f8 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -129,6 +129,9 @@ typedef struct SSyncFSM { void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); + void (*FpBecomeLeaderCb)(struct SSyncFSM* pFsm); + void (*FpBecomeFollowerCb)(struct SSyncFSM* pFsm); + int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); int32_t (*FpGetSnapshotInfo)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index eab6e5561f..3ca6978156 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -284,11 +284,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) +#define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) +#define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) @@ -619,6 +621,7 @@ int32_t* taosGetErrno(); //tmq #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) +#define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #ifdef __cplusplus } diff --git a/source/client/src/TSDBJNIConnector.c b/source/client/src/TSDBJNIConnector.c index 38bbec24ce..37041da695 100644 --- a/source/client/src/TSDBJNIConnector.c +++ b/source/client/src/TSDBJNIConnector.c @@ -611,65 +611,6 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeConnectionIm } } -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp(JNIEnv *env, jobject jobj, jlong con, - jboolean restart, jstring jtopic, - jstring jsql, jint jinterval) { - jlong sub = 0; - TAOS *taos = (TAOS *)con; - char *topic = NULL; - char *sql = NULL; - - jniGetGlobalMethod(env); - jniDebug("jobj:%p, in TSDBJNIConnector_subscribeImp", jobj); - - if (jtopic != NULL) { - topic = (char *)(*env)->GetStringUTFChars(env, jtopic, NULL); - } - if (jsql != NULL) { - sql = (char *)(*env)->GetStringUTFChars(env, jsql, NULL); - } - - if (topic == NULL || sql == NULL) { - jniDebug("jobj:%p, invalid argument: topic or sql is NULL", jobj); - return sub; - } - - TAOS_SUB *tsub = taos_subscribe(taos, (int)restart, topic, sql, NULL, NULL, jinterval); - sub = (jlong)tsub; - - if (sub == 0) { - jniDebug("jobj:%p, failed to subscribe: topic:%s", jobj, topic); - } else { - jniDebug("jobj:%p, successfully subscribe: topic: %s", jobj, topic); - } - - (*env)->ReleaseStringUTFChars(env, jtopic, topic); - (*env)->ReleaseStringUTFChars(env, jsql, sql); - - return sub; -} - -JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { - jniDebug("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%lld", jobj, sub); - jniGetGlobalMethod(env); - - TAOS_SUB *tsub = (TAOS_SUB *)sub; - TAOS_RES *res = taos_consume(tsub); - - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return 0l; - } - - return (jlong)res; -} - -JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, - jboolean keepProgress) { - TAOS_SUB *tsub = (TAOS_SUB *)sub; - taos_unsubscribe(tsub, keepProgress); -} - JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_validateCreateTableSqlImp(JNIEnv *env, jobject jobj, jlong con, jbyteArray jsql) { TAOS *tscon = (TAOS *)con; @@ -1087,4 +1028,4 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsert return JNI_OUT_OF_MEMORY; } return (jlong)tres; -} \ No newline at end of file +} diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0d41d35721..0ec724c6d0 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -939,21 +939,6 @@ const void *taos_get_raw_block(TAOS_RES *res) { return pRequest->body.resInfo.pData; } -TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, - void *param, int interval) { - // TODO - return NULL; -} - -TAOS_RES *taos_consume(TAOS_SUB *tsub) { - // TODO - return NULL; -} - -void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { - // TODO -} - int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 24997a735c..ea7f03a416 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -122,6 +122,7 @@ enum { TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__NO_TOPIC, + TMQ_CONSUMER_STATUS__RECOVER, }; enum { @@ -134,10 +135,8 @@ typedef struct { // statistics int64_t pollCnt; // offset - /*int64_t committedOffset;*/ - /*int64_t currentOffset;*/ - STqOffsetVal committedOffsetNew; - STqOffsetVal currentOffsetNew; + STqOffsetVal committedOffset; + STqOffsetVal currentOffset; // connection info int32_t vgId; int32_t vgStatus; @@ -152,7 +151,6 @@ typedef struct { SArray* vgs; // SArray - int8_t isSchemaAdaptive; SSchemaWrapper schema; } SMqClientTopic; @@ -190,10 +188,9 @@ typedef struct { } SMqPollCbParam; typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - /*int8_t freeOffsets;*/ + tmq_t* tmq; + int8_t automatic; + int8_t async; int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -207,7 +204,7 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; STqOffset* pOffset; -} SMqCommitCbParam2; +} SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); @@ -215,6 +212,7 @@ tmq_conf_t* tmq_conf_new() { conf->autoCommit = true; conf->autoCommitInterval = 5000; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; + conf->hbBgEnable = true; return conf; } @@ -371,8 +369,8 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } -int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) { - SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; +int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; // push into array #if 0 @@ -418,7 +416,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pOffset->val = pVg->currentOffsetNew; + pOffset->val = pVg->currentOffset; int32_t groupLen = strlen(tmq->groupId); memcpy(pOffset->subKey, tmq->groupId, groupLen); @@ -443,7 +441,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT tEncodeSTqOffset(&encoder, pOffset); // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + SMqCommitCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam)); pParam->params = pParamSet; pParam->pOffset = pOffset; @@ -462,13 +460,13 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pVg->vgId, pOffset->val.version); // TODO: put into cb - pVg->committedOffsetNew = pVg->currentOffsetNew; + pVg->committedOffset = pVg->currentOffset; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->fp = tmqCommitCb; pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; // send msg @@ -504,7 +502,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->tmq = tmq; pParamSet->automatic = 0; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -518,7 +515,7 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (pVg->vgId != vgId) continue; - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { goto FAIL; } @@ -550,8 +547,8 @@ FAIL: return 0; } -int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, - void* userParam) { +int32_t tmqCommitInner(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, + void* userParam) { int32_t code = -1; if (msg != NULL) { @@ -566,7 +563,6 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -583,7 +579,9 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgId:%d", tmq->consumerId, pTopic->topicName, pVg->vgId); - if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { + tscDebug("consumer: %ld, vg:%d, current %ld, committed %ld", tmq->consumerId, pVg->vgId, + pVg->currentOffset.version, pVg->committedOffset.version); if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { continue; } @@ -699,7 +697,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { tmqAskEp(tmq, true); taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { - tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); + tmqCommitInner(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else { @@ -888,12 +886,6 @@ FAIL: return NULL; } -#if 0 -int32_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { - return tmqCommitInner2(tmq, offsets, 0, async, tmq->commitCb, tmq->commitCbUserParam); -} -#endif - int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -967,7 +959,11 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { code = param.rspErr; if (code != 0) goto FAIL; + int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + goto FAIL; + } tscDebug("consumer not ready, retry"); taosMsleep(500); } @@ -1006,8 +1002,12 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t epoch = pParam->epoch; taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); - if (pMsg->pData) taosMemoryFreeClear(pMsg->pData); + tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); + if (pMsg->pData) taosMemoryFree(pMsg->pData); + if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); + goto CREATE_MSG_FAIL; + } if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { @@ -1083,7 +1083,7 @@ CREATE_MSG_FAIL: return -1; } -bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { +bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); @@ -1112,10 +1112,10 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffsetNew); + tFormatOffset(buf, 80, &pVgCur->currentOffset); tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(STqOffsetVal)); } } } @@ -1142,7 +1142,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew = offsetNew, + .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1166,93 +1166,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 0 -bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { - /*printf("call update ep %d\n", epoch);*/ - bool set = false; - int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, - topicNumGet); - SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); - if (newTopics == NULL) { - return false; - } - SHashObj* pHash = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); - if (pHash == NULL) { - taosArrayDestroy(newTopics); - return false; - } - - // find topic, build hash - for (int32_t i = 0; i < topicNumGet; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(pRsp->topics, i); - topic.schema = pTopicEp->schema; - taosHashClear(pHash); - topic.topicName = strdup(pTopicEp->topic); - tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - - tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); - int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for (int32_t j = 0; j < topicNumCur; j++) { - // find old topic - SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { - int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); - if (vgNumCur == 0) break; - for (int32_t k = 0; k < vgNumCur; k++) { - SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); - sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); - } - break; - } - } - - int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgNumGet; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = pVgEp->offset; - tscDebug("consumer:%" PRId64 ", (epoch %d) original offset of vgId:%d is %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - if (pOffset != NULL) { - offset = *pOffset; - tscDebug("consumer:%" PRId64 ", (epoch %d) receive offset of vgId:%d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, - vgKey); - } - tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); - SMqClientVg clientVg = { - .pollCnt = 0, - .currentOffset = offset, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, - .vgSkipCnt = 0, - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(newTopics, &topic); - } - if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); - taosHashCleanup(pHash); - tmq->clientTopics = newTopics; - - if (taosArrayGetSize(tmq->clientTopics) == 0) - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC); - else - atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); - - atomic_store_32(&tmq->epoch, epoch); - return set; -} -#endif - int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; tmq_t* tmq = pParam->tmq; @@ -1278,7 +1191,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ - tmqUpdateEp2(tmq, head->epoch, &rsp); + tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); @@ -1401,17 +1314,6 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { #endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - /*int64_t reqOffset;*/ - /*if (pVg->currentOffset >= 0) {*/ - /*reqOffset = pVg->currentOffset;*/ - /*} else {*/ - /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ - /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ - /*return NULL;*/ - /*}*/ - /*reqOffset = tmq->resetOffsetCfg;*/ - /*}*/ - SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { return NULL; @@ -1430,7 +1332,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffsetNew; + pReq->reqOffset = pVg->currentOffset; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1534,7 +1436,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { /*printf("send poll\n");*/ char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); + tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ @@ -1552,7 +1454,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp2(tmq, rspWrapper->epoch, rspMsg); + tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ *pReset = true; } else { @@ -1586,7 +1488,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1609,8 +1511,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vgId:%d, offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, * rspMsg->msg.rspOffset);*/ - pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; - pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; + pVg->currentOffset.version = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffset.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1653,6 +1555,17 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { + int32_t retryCnt = 0; + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) { + if (retryCnt++ > 10) { + return NULL; + } + tscDebug("consumer not ready, retry"); + taosMsleep(500); + } + } + while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) return NULL; @@ -3384,10 +3297,10 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { // - tmqCommitInner2(tmq, msg, 0, 1, cb, param); + tmqCommitInner(tmq, msg, 0, 1, cb, param); } int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { // - return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); + return tmqCommitInner(tmq, msg, 0, 0, NULL, NULL); } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index ce80942356..16681fb705 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -135,12 +135,12 @@ static const SSysDbTableSchema streamSchema[] = { {.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "watermark", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, - {.name = "trigger", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "trigger", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SSysDbTableSchema userTblsSchema[] = { @@ -284,8 +284,7 @@ static const SSysDbTableSchema consumerSchema[] = { {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topics", .bytes = TSDB_TOPIC_FNAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, - {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, + /*{.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},*/ {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b630419fc4..7dd3ce34c3 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -305,7 +305,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) taosArrayPush(desc.subDesc, &sDesc); } } - + ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); taosArrayPush(pReq->query->queryDesc, &desc); @@ -5770,3 +5770,40 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { } return 0; } + +int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->ts) < 0) return -1; + return 0; +} + +int32_t tDecodeSSingleDeleteReq(SDecoder *pDecoder, SSingleDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->ts) < 0) return -1; + return 0; +} + +int32_t tEncodeSBatchDeleteReq(SEncoder *pEncoder, const SBatchDeleteReq *pReq) { + if (tEncodeI64(pEncoder, pReq->suid) < 0) return -1; + int32_t sz = taosArrayGetSize(pReq->deleteReqs); + if (tEncodeI32(pEncoder, sz) < 0) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(pReq->deleteReqs, i); + if (tEncodeSSingleDeleteReq(pEncoder, pOneReq) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { + if (tDecodeI64(pDecoder, &pReq->suid) < 0) return -1; + int32_t sz; + if (tDecodeI32(pDecoder, &sz) < 0) return -1; + pReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + if (pReq->deleteReqs == NULL) return -1; + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq deleteReq; + if (tDecodeSSingleDeleteReq(pDecoder, &deleteReq) < 0) return -1; + taosArrayPush(pReq->deleteReqs, &deleteReq); + } + return 0; +} diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 6f58a0ded1..565498a47b 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -32,9 +32,13 @@ const uint8_t tdVTypeByte[2][3] = {{ }; // declaration -static uint8_t tdGetBitmapByte(uint8_t byte); -static int32_t tdCompareColId(const void *arg1, const void *arg2); -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2); +static uint8_t tdGetBitmapByte(uint8_t byte); +static bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +static bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal); +static bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, + col_id_t colIdx, SCellVal *pVal); +static bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal); +static void tdSCellValPrint(SCellVal *pVal, int8_t colType); // implementation /** @@ -330,14 +334,14 @@ void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>type:%d,sver:%d ", tag, (int32_t)TD_ROW_TYPE(row), (int32_t)TD_ROW_SVER(row)); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + STColumn *cols = (STColumn *)&iter.pSchema->columns; + while (true) { + SCellVal sVal = {.valType = 255, NULL}; + if (!tdSTSRowIterNext(&iter, &sVal)) { break; } ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); + tdSCellValPrint(&sVal, cols[iter.colIdx - 1].type); } printf("\n"); } @@ -420,6 +424,16 @@ void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { + return 1; + } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, col_id_t colIdx, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); @@ -456,7 +470,7 @@ bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t fl return true; } -bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { +bool tdSTSRowIterFetch(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pVal->val = &pIter->pRow->ts; pVal->valType = TD_VTYPE_NORM; @@ -477,10 +491,10 @@ bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCe return false; } } - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + return tdSTSRowIterGetKvVal(pIter, colId, &pIter->kvIdx, pVal); } else { pVal->valType = TD_VTYPE_NONE; terrno = TSDB_CODE_INVALID_PARA; @@ -489,13 +503,68 @@ bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCe return true; } -bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { +bool tdSTSRowIterNext(STSRowIter *pIter, SCellVal *pVal) { + if (pIter->colIdx >= pIter->pSchema->numOfCols) { + return false; + } + + STColumn *pCol = &pIter->pSchema->columns[pIter->colIdx]; + + if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + ++pIter->colIdx; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + tdSTSRowIterGetTpVal(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + } else if (TD_IS_KV_ROW(pIter->pRow)) { + tdSTSRowIterGetKvVal(pIter, pCol->colId, &pIter->kvIdx, pVal); + } else { + ASSERT(0); + } + ++pIter->colIdx; + + return true; +} + +bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + if (pRow->statis == 0) { + pVal->valType = TD_VTYPE_NORM; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + return TSDB_CODE_SUCCESS; + } + + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + return terrno; + } + + if (pVal->valType == TD_VTYPE_NORM) { + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } + + return true; +} + +bool tdSTSRowIterGetKvVal(STSRowIter *pIter, col_id_t colId, col_id_t *nIdx, SCellVal *pVal) { STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + void *pColIdx = TD_ROW_COL_IDX(pRow); while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, *nIdx * sizeof(SKvRowIdx)); if (pKvIdx->colId == colId) { ++(*nIdx); pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); @@ -518,48 +587,13 @@ bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, } } -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(pKvIdx, TD_ROW_COL_IDX(pRow)) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->kvIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif return true; } -bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { - return 1; - } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { - return -1; - } else { - return 0; - } -} - int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { STColumn *pTColumn; SColVal *pColVal; @@ -625,7 +659,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { if (maxVarDataLen > 0) { varBuf = taosMemoryMalloc(maxVarDataLen); if (!varBuf) { - if(isAlloc) { + if (isAlloc) { taosMemoryFreeClear(*ppRow); } terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -673,6 +707,19 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { return 0; } +static FORCE_INLINE int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; + STColumn *pCol = (STColumn *)arg2; + + if (colId < pCol->colId) { + return -1; + } else if (colId == pCol->colId) { + return 0; + } else { + return 1; + } +} + bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { pVal->val = &pIter->pRow->ts; @@ -712,19 +759,6 @@ bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCell return true; } -static int32_t tdCompareColId(const void *arg1, const void *arg2) { - int32_t colId = *(int32_t *)arg1; - STColumn *pCol = (STColumn *)arg2; - - if (colId < pCol->colId) { - return -1; - } else if (colId == pCol->colId) { - return 0; - } else { - return 1; - } -} - int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); @@ -938,7 +972,7 @@ int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colTyp break; case TD_VTYPE_NONE: if (!pBuilder->hasNone) pBuilder->hasNone = true; - break; + return TSDB_CODE_SUCCESS; default: ASSERT(0); break; @@ -970,13 +1004,11 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo STSRow *row = pBuilder->pBuf; // No need to store None/Null values. + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN if (valType == TD_VTYPE_NORM) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); if (IS_VAR_DATA_TYPE(colType)) { if (isCopyVarData) { memcpy(ptr, val, varDataTLen(val)); @@ -987,26 +1019,6 @@ int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const vo TD_ROW_LEN(row) += TYPE_BYTES[colType]; } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif return 0; } @@ -1044,24 +1056,6 @@ int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const vo memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); } } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif return 0; } @@ -1329,7 +1323,7 @@ void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { pIter->pRow = pRow; pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->colIdx = 0; // PRIMARYKEY_TIMESTAMP_COL_ID; pIter->kvIdx = 0; } @@ -1367,4 +1361,4 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, value); } -} +} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 1b799b1e5e..d13daffa08 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -364,6 +364,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_CHECK_ALTER_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index c5c4800e02..9f516904ce 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -32,6 +32,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic); SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb); const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 39b57f2966..614348c209 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -131,8 +131,9 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); - if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { + if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { mndReleaseConsumer(pMnode, pConsumer); + terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -275,6 +276,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -305,15 +307,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); -#if 1 atomic_store_32(&pConsumer->hbStatus, 0); -#endif // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + mInfo("try to recover consumer %ld", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -326,6 +327,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #endif if (status != MQ_CONSUMER_STATUS__READY) { + mInfo("consumer %ld not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -939,13 +941,9 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, NULL, true); } - // pid - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true); - // end point - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true); + /*pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);*/ + /*colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);*/ // up time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 89ea7abc23..9dde083f50 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -995,11 +995,13 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndCheckTopicExist(pMnode, pDb) < 0) goto _OVER; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; - if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; @@ -1706,7 +1708,7 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - int64_t uid = *(int64_t*)p2; + int64_t uid = *(int64_t *)p2; if (pVgroup->dbUid == uid) { *numOfTables += pVgroup->numOfTables; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7bde05aca5..8c453e0c88 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -197,6 +197,30 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) { sdbRelease(pSdb, pStream); } +static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { + int8_t status = atomic_load_8(&pStream->status); + if (status == STREAM_STATUS__NORMAL) { + strcpy(dst, "normal"); + } else if (status == STREAM_STATUS__STOP) { + strcpy(dst, "stop"); + } else if (status == STREAM_STATUS__FAILED) { + strcpy(dst, "failed"); + } else if (status == STREAM_STATUS__RECOVER) { + strcpy(dst, "recover"); + } +} + +static void mndShowStreamTrigger(char *dst, SStreamObj *pStream) { + int8_t trigger = pStream->trigger; + if (trigger == STREAM_TRIGGER_AT_ONCE) { + strcpy(dst, "at once"); + } else if (trigger == STREAM_TRIGGER_WINDOW_CLOSE) { + strcpy(dst, "window close"); + } else if (trigger == STREAM_TRIGGER_MAX_DELAY) { + strcpy(dst, "max delay"); + } +} + static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->sourceDB[0] == 0 || pCreate->targetStbFullName[0] == 0) { @@ -837,7 +861,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { sdbCancelFetch(pSdb, pIter); mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64, pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid); - terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; return -1; } else { #if 0 @@ -926,23 +950,46 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); + char status[20 + VARSTR_HEADER_SIZE] = {0}; + mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); + varDataSetLen(status, strlen(varDataVal(status))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->status, true); + colDataAppend(pColInfo, numOfRows, (const char *)&status, false); + char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(sourceDB)); + varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->sourceDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); + char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&n, varDataVal(targetDB)); + varDataSetLen(targetDB, strlen(varDataVal(targetDB))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetDb, true); + colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->targetSTbName, true); + if (pStream->targetSTbName[0] == 0) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, NULL, true); + } else { + char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); + varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); + char trigger[20 + VARSTR_HEADER_SIZE] = {0}; + mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); + varDataSetLen(trigger, strlen(varDataVal(trigger))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pStream->trigger, false); + colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); numOfRows++; sdbRelease(pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8feed476cd..3f310ee9c0 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -398,13 +398,27 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR } } - // 8. TODO generate logs - mInfo("rebalance calculation completed, rebalanced vg:"); + // 8. generate logs + mInfo("mq rebalance: calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, + mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } + { + void *pIter = NULL; + while (1) { + pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); + if (pIter == NULL) break; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + int32_t sz = taosArrayGetSize(pConsumerEp->vgs); + mInfo("mq rebalance: final cfg: consumer %ld has %d vg", pConsumerEp->consumerId, sz); + for (int32_t i = 0; i < sz; i++) { + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); + mInfo("mq rebalance: final cfg: vg %d to consumer %ld", pVgEp->vgId, pConsumerEp->consumerId); + } + } + } // 9. clear taosHashCleanup(pHash); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index b8ff7be46a..37d5aeb62d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -166,6 +166,18 @@ void mndLeaderTransfer(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb mDebug("vgId:1, mnode leader transfer finish"); } +static void mndBecomeFollower(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become follower"); + + // clear old leader resource +} + +static void mndBecomeLeader(struct SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + mDebug("vgId:1, become leader"); +} + SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pMnode; @@ -175,6 +187,8 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpRestoreFinishCb = mndRestoreFinish; pFsm->FpLeaderTransferCb = mndLeaderTransfer; pFsm->FpReConfigCb = mndReConfig; + pFsm->FpBecomeLeaderCb = mndBecomeLeader; + pFsm->FpBecomeFollowerCb = mndBecomeFollower; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index f343c2e055..820bb4b636 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -782,6 +782,26 @@ static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +int32_t mndCheckTopicExist(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqTopicObj *pTopic = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->dbUid == pDb->uid) { + sdbRelease(pSdb, pTopic); + terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + return -1; + } + + sdbRelease(pSdb, pTopic); + } + return 0; +} + int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 2b154fce04..43bb92ec23 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -157,7 +157,7 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); int32_t tqProcessCheckAlterInfoReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen); @@ -171,8 +171,8 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list); -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId); +SSubmitReq* tdBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, + const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq); // sma int32_t smaInit(); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 7f69acc889..f46d9dc29c 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -119,7 +119,7 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { SName stbFullName = {0}; tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); SVCreateStbReq pReq = {0}; - pReq.name = (char*)tNameGetTableName(&stbFullName); + pReq.name = (char *)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; @@ -200,8 +200,10 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { goto _err; } - SSubmitReq *pSubmitReq = tdBlockToSubmit((const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, - pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId); + SBatchDeleteReq deleteReq; + SSubmitReq *pSubmitReq = + tdBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, true, pTsmaStat->pTSma->dstTbUid, + pTsmaStat->pTSma->dstTbName, pTsmaStat->pTSma->dstVgId, &deleteReq); if (!pSubmitReq) { smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), @@ -230,4 +232,4 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { _err: tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02ca248054..112543e340 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -183,7 +183,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } -int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen, int64_t ver) { STqOffset offset = {0}; SDecoder decoder; tDecoderInit(&decoder, msg, msgLen); @@ -302,6 +302,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6ce8dbe5d9..5d7814a045 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -325,7 +325,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { for (int32_t i = 0; i < colActual; i++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { + if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } if (colDataAppend(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index d2bc95d166..42fb5c329d 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -13,10 +13,44 @@ * along with this program. If not, see . */ +#include "tcommon.h" +#include "tmsg.h" #include "tq.h" -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { +int32_t tdBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, + SBatchDeleteReq* deleteReq) { + ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); + int32_t totRow = pDataBlock->info.rows; + SColumnInfoData* pTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pGidCol = taosArrayGet(pDataBlock->pDataBlock, GROUPID_COLUMN_INDEX); + for (int32_t row = 0; row < totRow; row++) { + int64_t ts = *(int64_t*)colDataGetData(pTsCol, row); + /*int64_t groupId = *(int64_t*)colDataGetData(pGidCol, row);*/ + int64_t groupId = 0; + char* name = buildCtbNameByGroupId(stbFullName, groupId); + tqDebug("stream delete msg: groupId :%ld, name: %s", groupId, name); + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, name) < 0) { + metaReaderClear(&mr); + taosMemoryFree(name); + return -1; + } + + int64_t uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFree(name); + SSingleDeleteReq req = { + .ts = ts, + .uid = uid, + }; + taosArrayPush(deleteReq->deleteReqs, &req); + } + return 0; +} + +SSubmitReq* tdBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, bool createTb, + int64_t suid, const char* stbFullName, int32_t vgId, SBatchDeleteReq* pDeleteReq) { SSubmitReq* ret = NULL; SArray* schemaReqs = NULL; SArray* schemaReqSz = NULL; @@ -33,10 +67,17 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + int32_t padding1 = 0; + void* padding2 = taosMemoryMalloc(1); + taosArrayPush(schemaReqSz, &padding1); + taosArrayPush(schemaReqs, &padding2); + } + + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, }; STag* pTag = NULL; taosArrayClear(tagArray); @@ -94,7 +135,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo int32_t cap = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + continue; + } + int32_t rows = pDataBlock->info.rows; // TODO min int32_t rowSize = pDataBlock->info.rowSize; int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); @@ -116,6 +160,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + tdBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } blkHead->numOfRows = htonl(pDataBlock->info.rows); blkHead->sversion = htonl(pTSchema->version); @@ -177,17 +226,47 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; + const SArray* pRes = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + SBatchDeleteReq deleteReq = {0}; tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); ASSERT(pTask->tbSink.pTSchema); - SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, - pTask->tbSink.stbFullName, pVnode->config.vgId); + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + SSubmitReq* pReq = tdBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, true, pTask->tbSink.stbUid, + pTask->tbSink.stbFullName, pVnode->config.vgId, &deleteReq); tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); + int32_t code; + int32_t len; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* buf = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + + ((SMsgHead*)buf)->vgId = pVnode->config.vgId; + + if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = buf, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + taosArrayDestroy(deleteReq.deleteReqs); + /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ // build write msg SRpcMsg msg = { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 468370b1a7..1da65288d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -182,7 +182,7 @@ static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { if (IS_VAR_DATA_TYPE(pCol->info.type)) { pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); - tsdbInfo("-------------------%d\n", pCol->info.bytes); + // tsdbInfo("-------------------%d\n", pCol->info.bytes); } } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b8e94de115..ecff58f3b1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -29,6 +29,7 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; @@ -144,7 +145,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp int32_t len; int32_t ret; - vTrace("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), + vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); pVnode->state.applied = version; @@ -190,6 +191,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; + case TDMT_VND_BATCH_DEL: + if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; + break; /* TQ */ case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), @@ -204,7 +208,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; case TDMT_VND_MQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + pMsg->contLen - sizeof(SMsgHead), version) < 0) { goto _err; } break; @@ -1053,6 +1057,24 @@ static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void return 0; } +static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SBatchDeleteReq deleteReq; + SDecoder decoder; + tDecoderInit(&decoder, pReq, len); + tDecodeSBatchDeleteReq(&decoder, &deleteReq); + + int32_t sz = taosArrayGetSize(deleteReq.deleteReqs); + for (int32_t i = 0; i < sz; i++) { + SSingleDeleteReq *pOneReq = taosArrayGet(deleteReq.deleteReqs, i); + int32_t code = tsdbDeleteTableData(pVnode->pTsdb, version, deleteReq.suid, pOneReq->uid, pOneReq->ts, pOneReq->ts); + if (code) { + // TODO + } + } + taosArrayDestroy(deleteReq.deleteReqs); + return 0; +} + static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index c525d09057..7ac124fdd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -672,6 +672,18 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vDebug("vgId:%d, sync restore finished", pVnode->config.vgId); } +static void vnodeBecomeFollower(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become follower", pVnode->config.vgId); + + // clear old leader resource +} + +static void vnodeBecomeLeader(struct SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vDebug("vgId:%d, become leader", pVnode->config.vgId); +} + static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { SSyncFSM *pFsm = taosMemoryCalloc(1, sizeof(SSyncFSM)); pFsm->data = pVnode; @@ -681,6 +693,8 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshot; pFsm->FpRestoreFinishCb = vnodeRestoreFinish; pFsm->FpLeaderTransferCb = vnodeLeaderTransfer; + pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; + pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; pFsm->FpReConfigCb = vnodeSyncReconfig; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 1ca60c89cd..7b81343358 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -883,6 +883,32 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } +int32_t ctgHashValueComp(void const *lp, void const *rp) { + uint32_t *key = (uint32_t *)lp; + SVgroupInfo *pVg = *(SVgroupInfo **)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + +int ctgVgInfoComp(const void* lp, const void* rp) { + SVgroupInfo *pLeft = *(SVgroupInfo **)lp; + SVgroupInfo *pRight = *(SVgroupInfo **)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + + int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -923,9 +949,19 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo } } + taosHashCancelIterate(dbInfo->vgHash, pIter); return TSDB_CODE_SUCCESS; } + SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(pVgList, &pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(pVgList, ctgVgInfoComp); + char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -940,25 +976,20 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); - void *pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - vgInfo = pIter; - if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgHash, pIter); - break; - } - - pIter = taosHashIterate(dbInfo->vgHash, pIter); - vgInfo = NULL; - } + SVgroupInfo **p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); - if (NULL == vgInfo) { + if (NULL == p) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + ASSERT(0); + taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } + vgInfo = *p; + SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pNewVg) { + taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -977,6 +1008,8 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo } } + taosArrayDestroy(pVgList); + CTG_RET(code); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d460644633..75db151f85 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -52,13 +52,6 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) -#define START_TS_COLUMN_INDEX 0 -#define END_TS_COLUMN_INDEX 1 -#define UID_COLUMN_INDEX 2 -#define GROUPID_COLUMN_INDEX 3 -#define CALCULATE_START_TS_COLUMN_INDEX 4 -#define CALCULATE_END_TS_COLUMN_INDEX 5 - enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, @@ -741,6 +734,7 @@ typedef struct STimeSliceOperatorInfo { SArray* pPrevRow; // SArray SArray* pNextRow; // SArray SArray* pLinearInfo; // SArray + bool fillLastPoint; bool isPrevRowSet; bool isNextRowSet; int32_t fillType; // fill type @@ -1017,6 +1011,7 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid); diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index c2de48d0eb..1e95f4a04f 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -37,7 +37,6 @@ typedef struct SFillLinearInfo { SPoint start; SPoint end; bool hasNull; - bool fillLastPoint; int16_t type; int32_t bytes; } SFillLinearInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index d3ced810c6..9e485e7684 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1277,8 +1277,12 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { } void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { + bool init = false; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, rowEntryInfoOffset); + if (init) { + continue; + } struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { @@ -1295,6 +1299,8 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } else { pResInfo->initialized = true; } + } else { + init = true; } } } @@ -1943,6 +1949,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pWrapper->exchangeId); if (pExchangeInfo == NULL) { qWarn("failed to acquire exchange operator, since it may have been released"); + taosMemoryFree(pMsg->pData); return TSDB_CODE_SUCCESS; } @@ -1963,6 +1970,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows); } else { + taosMemoryFree(pMsg->pData); pSourceDataInfo->code = code; qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); } @@ -3416,6 +3424,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n } int32_t code = createDiskbasedBuf(&pAggSup->pResultBuf, defaultPgsz, defaultBufsz, pKey, tsTempDir); if (code != TSDB_CODE_SUCCESS) { + qError("Create agg result buf failed since %s", tstrerror(code)); return code; } @@ -3435,7 +3444,11 @@ int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf return code; } - doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + code = doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].pBuf = pAggSup->pResultBuf; } diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 2f9eff50b6..7d2b84d0f0 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -132,6 +132,8 @@ void destroyMergeJoinOperator(void* param, int32_t numOfOutput) { SJoinOperatorInfo* pJoinOperator = (SJoinOperatorInfo*)param; nodesDestroyNode(pJoinOperator->pCondAfterMerge); + pJoinOperator->pRes = blockDataDestroy(pJoinOperator->pRes); + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a211542de1..d8de8df163 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -319,6 +319,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return TSDB_CODE_SUCCESS; } else { + qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); *status = FUNC_DATA_REQUIRED_DATA_LOAD; } } @@ -326,20 +327,19 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); // try to filter data block according to sma info - if (pTableScanInfo->pFilterNode != NULL) { - if (!loadSMA) { - doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); - } + if (pTableScanInfo->pFilterNode != NULL && (!loadSMA)) { + bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); + if (success) { + size_t size = taosArrayGetSize(pBlock->pDataBlock); + bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, size, pBlockInfo->rows); + if (!keep) { + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + pCost->filterOutBlocks += 1; + (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - bool keep = doFilterByBlockSMA(pTableScanInfo->pFilterNode, pBlock->pBlockAgg, taosArrayGetSize(pBlock->pDataBlock), - pBlockInfo->rows); - if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - pCost->filterOutBlocks += 1; - (*status) = FUNC_DATA_REQUIRED_FILTEROUT; - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; + } } } @@ -1174,10 +1174,15 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + bool isClosed = false; + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + if (isOverdue(tsCol[rowId], &pInfo->twAggSup)) { + win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); + isClosed = isCloseWindow(&win, &pInfo->twAggSup); + } // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); - if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup) && + if ((update || (isSignleIntervalWindow(pInfo) && isClosed && isDeletedWindow(&win, pBlock->info.groupId, pInfo->sessionSup.pIntervalAggSup))) && out) { appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid); } @@ -2458,81 +2463,6 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; -#if 0 - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->exprSupp.pExprInfo[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. -#endif - STagScanInfo* pInfo = pOperator->info; SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; SSDataBlock* pRes = pInfo->pRes; @@ -2610,6 +2540,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); + taosArrayDestroy(pInfo->pColMatchInfo); + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9198862ba3..2b67f2bf01 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -851,23 +851,34 @@ static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t g return TSDB_CODE_SUCCESS; } +static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) { + SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + if (newPos == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + newPos->groupId = groupId; + newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; + *(int64_t*)newPos->key = ts; + SWinRes key = {.ts = ts, .groupId = groupId}; + if (taosHashPut(pUpdatedMap, &key, sizeof(SWinRes), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) { + taosMemoryFree(newPos); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) { + return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap);; +} + static int32_t saveResultRow(SResultRow* result, uint64_t groupId, SArray* pUpdated) { return saveResult(result->win.skey, result->pageId, result->offset, groupId, pUpdated); } -static void removeResult(SArray* pUpdated, SWinRes* pKey) { - int32_t size = taosArrayGetSize(pUpdated); - int32_t index = binarySearchCom(pUpdated, size, pKey, TSDB_ORDER_DESC, compareResKey); - if (index >= 0 && 0 == compareResKey(pKey, pUpdated, index)) { - taosArrayRemove(pUpdated, index); - } -} - -static void removeResults(SArray* pWins, SArray* pUpdated) { +static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { SWinRes* pW = taosArrayGet(pWins, i); - removeResult(pUpdated, pW); + taosHashRemove(pUpdatedMap, pW, sizeof(SWinRes)); } } @@ -894,11 +905,14 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) { return -1; } -static void removeDeleteResults(SArray* pUpdated, SArray* pDelWins) { - int32_t upSize = taosArrayGetSize(pUpdated); +static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { + if (!pUpdatedMap || taosHashGetSize(pUpdatedMap) == 0) { + return; + } int32_t delSize = taosArrayGetSize(pDelWins); - for (int32_t i = 0; i < upSize; i++) { - SResKeyPos* pResKey = taosArrayGetP(pUpdated, i); + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + SResKeyPos* pResKey = (SResKeyPos*)pIte; int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes); if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) { taosArrayRemove(pDelWins, index); @@ -914,7 +928,7 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SArray* pUpdated) { + int32_t scanFlag, SHashObj* pUpdatedMap) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -940,7 +954,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResultRow(pResult, tableGroupId, pUpdated); + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } } @@ -997,7 +1011,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveResultRow(pResult, tableGroupId, pUpdated); + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } @@ -1437,7 +1451,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } } -static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { +static int32_t getAllIntervalWindow(SHashObj* pHashMap, SHashObj* resWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1446,7 +1460,7 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowPosition* pPos = (SResultRowPosition*)pIte; - int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins); + int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1455,7 +1469,7 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { } static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SHashObj* pPullDataMap, SArray* closeWins, SArray* pRecyPages, + SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, SDiskbasedBuf* pDiscBuf) { qDebug("===stream===close interval window"); void* pIte = NULL; @@ -1487,7 +1501,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); + int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1577,11 +1591,14 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { break; } + // qInfo("===stream===%ld", pBlock->info.version); printDataBlock(pBlock, "single interval recv"); if (pBlock->info.type == STREAM_CLEAR) { @@ -1594,7 +1611,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } @@ -1617,17 +1634,24 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); } pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated, + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosHashCleanup(pUpdatedMap); + taosArraySort(pUpdated, resultrowComparAsc); + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - removeDeleteResults(pUpdated, pInfo->pDelWins); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows > 0) { return pInfo->pDelRes; @@ -1695,7 +1719,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { - if (!fmIsInvertible(pFCtx[i].functionId)) { + if (fmIsUserDefinedFunc(pFCtx[i].functionId) || !fmIsInvertible(pFCtx[i].functionId)) { return false; } } @@ -2088,8 +2112,10 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock pSliceInfo->isNextRowSet = true; } -static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { +static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex, + bool isLastRow) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + bool fillLastPoint = pSliceInfo->fillLastPoint; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); @@ -2097,16 +2123,22 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { - int64_t startKey = *(int64_t*)colDataGetData(pTsCol, rowIndex); - int64_t endKey = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - pLinearInfo->start.key = startKey; - pLinearInfo->end.key = endKey; + if (isLastRow) { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else if (fillLastPoint) { + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + } else { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); + pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex + 1); - char* val; - val = colDataGetData(pColInfoData, rowIndex); - memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); - val = colDataGetData(pColInfoData, rowIndex + 1); - memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + char* val; + val = colDataGetData(pColInfoData, rowIndex); + memcpy(pLinearInfo->start.val, val, pLinearInfo->bytes); + val = colDataGetData(pColInfoData, rowIndex + 1); + memcpy(pLinearInfo->end.val, val, pLinearInfo->bytes); + } pLinearInfo->hasNull = false; } else { @@ -2114,6 +2146,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } + pSliceInfo->fillLastPoint = isLastRow ? true : false; + } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, @@ -2270,7 +2304,7 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB } pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo)); - if (pInfo->pNextRow == NULL) { + if (pInfo->pLinearInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2284,15 +2318,20 @@ static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes); linearInfo.hasNull = false; - linearInfo.fillLastPoint = false; linearInfo.type = pColInfo->info.type; linearInfo.bytes = pColInfo->info.bytes; taosArrayPush(pInfo->pLinearInfo, &linearInfo); } + pInfo->fillLastPoint = false; + return TSDB_CODE_SUCCESS; } +static bool needToFillLastPoint(STimeSliceOperatorInfo* pSliceInfo) { + return (pSliceInfo->fillLastPoint == true && pSliceInfo->fillType == TSDB_FILL_LINEAR); +} + static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { int32_t code; code = initPrevRowsKeeper(pInfo, pBlock); @@ -2357,6 +2396,23 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t i = 0; i < pBlock->info.rows; ++i) { int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); + if (i == 0 && needToFillLastPoint(pSliceInfo)) { // first row in current block + doKeepLinearInfo(pSliceInfo, pBlock, i, false); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + } + if (ts == pSliceInfo->current) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; @@ -2375,9 +2431,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { // for linear interpolation, always fill value between this and next points; // if its the first point in data block, also fill values between previous(if there's any) and this point; - // if its the last point in data block, no need to fill, but reserve this point as the start value for next data block. + // if its the last point in data block, no need to fill, but reserve this point as the start value and do + // the interpolation when processing next data block. if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2397,6 +2454,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } + } else {// it is the last row of current block + //store ts value as start, and calculate interp value when processing next block + doKeepLinearInfo(pSliceInfo, pBlock, i, true); } } else { // non-linear interpolation pSliceInfo->current = @@ -2415,7 +2475,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); // no need to increate pSliceInfo->current here //pSliceInfo->current = // taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2495,7 +2555,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (pSliceInfo->fillType == TSDB_FILL_LINEAR) { - doKeepLinearInfo(pSliceInfo, pBlock, i); + doKeepLinearInfo(pSliceInfo, pBlock, i, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (i < pBlock->info.rows - 1) { @@ -2831,7 +2891,7 @@ STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { } static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SArray* pUpdated) { + SHashObj* pUpdatedMap) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -2913,8 +2973,8 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { - saveResultRow(pResult, tableGroupId, pUpdated); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); @@ -3020,6 +3080,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; @@ -3077,7 +3139,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { clearSpecialDataBlock(pInfo->pUpdateRes); - removeDeleteResults(pUpdated, pInfo->pDelWins); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); pOperator->status = OP_RES_TO_RETURN; qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); break; @@ -3104,7 +3166,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { taosArrayDestroy(pUpWins); continue; } - removeResults(pUpWins, pUpdated); + removeResults(pUpWins, pUpdatedMap); copyDataBlock(pInfo->pUpdateRes, pBlock); // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); pInfo->returnUpdate = true; @@ -3122,15 +3184,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated); continue; } - removeResults(pInfo->pDelWins, pUpdated); + removeResults(pInfo->pDelWins, pUpdatedMap); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinRes)); doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); - removeResults(pUpWins, pUpdated); + removeResults(pUpWins, pUpdatedMap); taosArrayDestroy(pUpWins); if (taosArrayGetSize(pUpdated) > 0) { break; @@ -3146,7 +3208,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdated); + doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3171,12 +3233,19 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, - pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } else { pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; } + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosHashCleanup(pUpdatedMap); + taosArraySort(pUpdated, resultrowComparAsc); + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 8f7ff04d92..e709643af9 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -13,38 +13,37 @@ * along with this program. If not, see . */ -#include "os.h" #include "tsimplehash.h" +#include "os.h" #include "taoserror.h" #define SHASH_DEFAULT_LOAD_FACTOR 0.75 -#define HASH_MAX_CAPACITY (1024*1024*16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) -#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl)) -#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode)) +#define GET_SHASH_NODE_KEY(_n, _dl) ((char *)(_n) + sizeof(SHNode) + (_dl)) +#define GET_SHASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHNode)) -#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR) +#define HASH_INDEX(v, c) ((v) & ((c)-1)) -#define FREE_HASH_NODE(_n) \ - do { \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_n) \ + do { \ + taosMemoryFreeClear(_n); \ } while (0); typedef struct SHNode { - struct SHNode *next; - char data[]; + struct SHNode *next; + char data[]; } SHNode; struct SSHashObj { - SHNode **hashList; - size_t capacity; // number of slots - int64_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _equal_fn_t equalFp; // equal function - int32_t keyLen; - int32_t dataLen; + SHNode **hashList; + size_t capacity; // number of slots + int64_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _equal_fn_t equalFp; // equal function + int32_t keyLen; + int32_t dataLen; }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -62,7 +61,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t capacity = 4; } - SSHashObj* pHashObj = (SSHashObj*) taosMemoryCalloc(1, sizeof(SSHashObj)); + SSHashObj *pHashObj = (SSHashObj *)taosMemoryCalloc(1, sizeof(SSHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -72,7 +71,7 @@ SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t pHashObj->capacity = taosHashCapacity((int32_t)capacity); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->keyLen = keyLen; @@ -91,7 +90,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); + return (int32_t)atomic_load_64((int64_t *)&pHashObj->size); } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -108,41 +107,42 @@ static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pDat } static void taosHashTableResize(SSHashObj *pHashObj) { - if (!HASH_NEED_RESIZE(pHashObj)) { + if (!SHASH_NEED_RESIZE(pHashObj)) { return; } int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u); if (newCapacity > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { -// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); + // qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; } size_t inc = newCapacity - pHashObj->capacity; - memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char *)pNewEntryList + pHashObj->capacity * sizeof(void *), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { - SHNode* pNode = pHashObj->hashList[idx]; - SHNode *pNext; - SHNode *pPrev = NULL; - + SHNode *pNode = pHashObj->hashList[idx]; if (pNode == NULL) { continue; } + SHNode *pNext; + SHNode *pPrev = NULL; + + while (pNode != NULL) { - void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); + void *key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen); uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen); int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity); @@ -166,8 +166,9 @@ static void taosHashTableResize(SSHashObj *pHashObj) { int64_t et = taosGetTimestampUs(); -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", + // (int32_t)pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { @@ -210,7 +211,7 @@ int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) { pNewNode->next = pHashObj->hashList[slot]; pHashObj->hashList[slot] = pNewNode; atomic_add_fetch_64(&pHashObj->size, 1); - } else { //update data + } else { // update data memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen); } @@ -230,9 +231,7 @@ static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void return pNode; } -static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { - return tSimpleHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) { return tSimpleHashGetSize(pHashObj) == 0; } void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) { @@ -299,9 +298,9 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj); } -void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { +void *tSimpleHashGetKey(const SSHashObj *pHashObj, void *data, size_t *keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = ((SHNode*)(char*)data - offset); + SHNode *node = ((SHNode *)(char *)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/tfunctionInt.h similarity index 76% rename from source/libs/function/inc/taggfunction.h rename to source/libs/function/inc/tfunctionInt.h index 669c2635b5..8f6cbc977e 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TAGGFUNCTION_H -#define TDENGINE_TAGGFUNCTION_H +#ifndef TDENGINE_TFUNCTIONINT_H +#define TDENGINE_TFUNCTIONINT_H #ifdef __cplusplus extern "C" { @@ -28,17 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" -#define AVG_FUNCTION_INTER_BUFFER_SIZE 50 - -#define DATA_SET_FLAG ',' // to denote the output area has data, not null value -#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) - -typedef struct SInterpInfoDetail { - TSKEY ts; // interp specified timestamp - int8_t type; - int8_t primaryCol; -} SInterpInfoDetail; - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** @@ -57,4 +46,4 @@ static FORCE_INLINE void initResultRowEntry(SResultRowEntryInfo *pResInfo, int32 } #endif -#endif // TDENGINE_TAGGFUNCTION_H +#endif // TDENGINE_TFUNCTIONINT_H diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index eb5d05c540..d44cb952e3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,10 +18,10 @@ #include "function.h" #include "query.h" #include "querynodes.h" -#include "taggfunction.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" +#include "tfunctionInt.h" #include "tglobal.h" #include "thistogram.h" #include "tpercentile.h" @@ -312,14 +312,6 @@ typedef struct SGroupKeyInfo { #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx* __ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - __ctx->fpSet.process(__ctx); \ - } \ - } while (0); - #define DO_UPDATE_SUBSID_RES(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ @@ -506,8 +498,7 @@ int32_t functionFinalizeWithResultBuf(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - cleanupResultRowEntry(pResInfo); + pResInfo->isNullRes = (pResInfo->isNullRes == 1) ? 1 : (pResInfo->numOfRes == 0);; char* in = finalResult; colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -4284,9 +4275,9 @@ static int32_t histogramFunctionImpl(SqlFunctionCtx* pCtx, bool isPartial) { } if (!isPartial) { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, pInfo->numOfBins); + GET_RES_INFO(pCtx)->numOfRes = pInfo->numOfBins; } else { - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + GET_RES_INFO(pCtx)->numOfRes = 1; } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c deleted file mode 100644 index c8998b9d94..0000000000 --- a/source/libs/function/src/taggfunction.c +++ /dev/null @@ -1,3910 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "thash.h" -#include "ttypes.h" - -#include "function.h" -#include "taggfunction.h" -#include "tbuffer.h" -#include "tcompression.h" -#include "thistogram.h" -#include "tpercentile.h" -#include "ttszip.h" -#include "tdatablock.h" -#include "tudf.h" - -#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) - -#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) -#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) - -#define GET_TRUE_DATA_TYPE() \ - int32_t type = 0; \ - if (pCtx->scanFlag == MERGE_STAGE) { \ - type = pCtx->resDataInfo.type; \ - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \ - } else { \ - type = pCtx->inputType; \ - } - -#define SET_VAL(ctx, numOfElem, res) \ - do { \ - if ((numOfElem) <= 0) { \ - break; \ - } \ - GET_RES_INFO(ctx)->numOfRes = (res); \ - } while (0) - -#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res)); - -#define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0) - -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ - aggFunc[FUNCTION_TAG].addInput(__ctx); \ - } \ - } while (0); - -void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} - -void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } - -typedef struct tValuePair { - SVariant v; - int64_t timestamp; - char * pTags; // the corresponding tags of each record in the final result -} tValuePair; - -typedef struct SSpreadInfo { - double min; - double max; - int8_t hasResult; -} SSpreadInfo; - -typedef struct SSumInfo { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; - int8_t hasResult; -} SSumInfo; - -// the attribute of hasResult is not needed since the num attribute would server as this purpose -typedef struct SAvgInfo { - double sum; - int64_t num; -} SAvgInfo; - -typedef struct SStddevInfo { - double avg; - int64_t num; - double res; - int8_t stage; -} SStddevInfo; - -typedef struct SStddevdstInfo { - int64_t num; - double res; -} SStddevdstInfo; - -typedef struct SFirstLastInfo { - int8_t hasResult; - TSKEY ts; -} SFirstLastInfo; - -typedef struct SFirstLastInfo SLastrowInfo; -typedef struct SPercentileInfo { - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; - -typedef struct STopBotInfo { - int32_t num; - tValuePair **res; -} STopBotInfo; - -// leastsquares do not apply to super table -typedef struct SLeastsquaresInfo { - double mat[2][3]; - double startVal; - int64_t num; -} SLeastsquaresInfo; - -typedef struct SAPercentileInfo { - SHistogramInfo *pHisto; -} SAPercentileInfo; - -typedef struct STSCompInfo { - STSBuf *pTSBuf; -} STSCompInfo; - -typedef struct SRateInfo { - double correctionValue; - double firstValue; - TSKEY firstKey; - double lastValue; - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - bool isIRate; // true for IRate functions, false for Rate functions -} SRateInfo; - -//typedef struct SDerivInfo { -// double prevValue; // previous value -// TSKEY prevTs; // previous timestamp -// bool ignoreNegative;// ignore the negative value -// int64_t tsWindow; // time window for derivative -// bool valueSet; // the value has been set already -//} SDerivInfo; - -typedef struct SResPair { - TSKEY key; - double avg; -} SResPair; - -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { - pCell->initialized = false; -} - -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { - int32_t maxRows = 0; - - for (int32_t j = 0; j < num; ++j) { -#if 0 - int32_t id = pCtx[j].functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (id == FUNCTION_TS || id == FUNCTION_TAG || id == FUNCTION_TAGPRJ) { - continue; - } -#endif - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { - maxRows = pResInfo->numOfRes; - } - } - - assert(maxRows >= 0); - - blockDataEnsureCapacity(pResBlock, maxRows); - for(int32_t i = 0; i < num; ++i) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); - if (pResInfo->numOfRes == 0) { - for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, NULL, true); // TODO add set null data api - } - } else { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); - } - } - } - - pResBlock->info.rows = maxRows; - return maxRows; -} - -void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { - for (int32_t j = 0; j < num; ++j) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); - pResInfo->numOfRes = 0; - } -} - -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { - assert(pEntry != NULL); - return pEntry->complete; -} - -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { - return pEntry->initialized; -} -#if 0 -int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, - bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { - if (!isValidDataType(dataType)) { -// qError("Illegal data type %d or data type length %d", dataType, dataBytes); - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - - if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG_DUMMY || - functionId == FUNCTION_DIFF || functionId == FUNCTION_PRJ || functionId == FUNCTION_TAGPRJ || - functionId == FUNCTION_TAG || functionId == FUNCTION_INTERP) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - if (functionId == FUNCTION_INTERP) { - pInfo->interBufSize = sizeof(SInterpInfoDetail); - } else { - pInfo->interBufSize = 0; - } - - return TSDB_CODE_SUCCESS; - } - - // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE - if (functionId == FUNCTION_TID_TAG) { // todo use struct - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_BLKINFO) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 16384; - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_COUNT) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_ARITHM) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = 0; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_TS_COMP) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = 1; // this results is compressed ts data, only one byte - pInfo->interBufSize = POINTER_BYTES; - return TSDB_CODE_SUCCESS; - } - - if (functionId == FUNCTION_DERIVATIVE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); // this results is compressed ts data, only one byte - pInfo->interBufSize = sizeof(SDerivInfo); - return TSDB_CODE_SUCCESS; - } - - if (isSuperTable) { -// if (functionId < 0) { -// if (pUdfInfo->bufSize > 0) { -// pInfo->type = TSDB_DATA_TYPE_BINARY; -// pInfo->bytes = pUdfInfo->bufSize; -// pInfo->interBufSize = pInfo->bytes; -// } else { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SUM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSumInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SAvgInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(SRateInfo); - pInfo->interBufSize = sizeof(SRateInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SSpreadInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes); - pInfo->interBufSize = pInfo->bytes; - - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(STwaInfo); - pInfo->interBufSize = pInfo->bytes; - return TSDB_CODE_SUCCESS; - } - } - - if (functionId == FUNCTION_SUM) { - if (IS_SIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_BIGINT; - } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) { - pInfo->type = TSDB_DATA_TYPE_UBIGINT; - } else { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - } - - pInfo->bytes = sizeof(int64_t); - pInfo->interBufSize = sizeof(SSumInfo); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_APERCT) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = - sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1); - return TSDB_CODE_SUCCESS; - } else if (functionId == FUNCTION_TWA) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(STwaInfo); - return TSDB_CODE_SUCCESS; - } - -// if (functionId < 0) { -// pInfo->type = pUdfInfo->resType; -// pInfo->bytes = pUdfInfo->resBytes; -// -// if (pUdfInfo->bufSize > 0) { -// pInfo->interBufSize = pUdfInfo->bufSize; -// } else { -// pInfo->interBufSize = pInfo->bytes; -// } -// -// return TSDB_CODE_SUCCESS; -// } - - if (functionId == FUNCTION_AVG) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SAvgInfo); - } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SRateInfo); - } else if (functionId == FUNCTION_STDDEV) { - pInfo->type = TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SStddevInfo); - } else if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes + DATA_SET_FLAG_SIZE; - } else if (functionId == FUNCTION_FIRST || functionId == FUNCTION_LAST) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - } else if (functionId == FUNCTION_SPREAD) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = sizeof(double); - pInfo->interBufSize = sizeof(SSpreadInfo); - } else if (functionId == FUNCTION_PERCT) { - pInfo->type = (int16_t)TSDB_DATA_TYPE_DOUBLE; - pInfo->bytes = (int16_t)sizeof(double); - pInfo->interBufSize = (int16_t)sizeof(SPercentileInfo); - } else if (functionId == FUNCTION_LEASTSQR) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); - pInfo->interBufSize = pInfo->bytes; - } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - - size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param; - - // the output column may be larger than sizeof(STopBotInfo) - pInfo->interBufSize = (int32_t)size; - } else if (functionId == FUNCTION_LAST_ROW) { - pInfo->type = (int16_t)dataType; - pInfo->bytes = (int16_t)dataBytes; - pInfo->interBufSize = dataBytes; - } else if (functionId == FUNCTION_STDDEV_DST) { - pInfo->type = TSDB_DATA_TYPE_BINARY; - pInfo->bytes = sizeof(SStddevdstInfo); - pInfo->interBufSize = (pInfo->bytes); - - } else { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - - return TSDB_CODE_SUCCESS; -} -#endif - -static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (pResultInfo->initialized) { - return false; - } - - memset(pCtx->pOutput, 0, (size_t)pCtx->resDataInfo.bytes); - initResultRowEntry(pResultInfo, pCtx->resDataInfo.interBufSize); - return true; -} -#if 0 -/** - * in handling the stable query, function_finalizer is called after the secondary - * merge being completed, during the first merge procedure, which is executed at the - * vnode side, the finalize will never be called. - * - * @param pCtx - */ -static void function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { // TODO set the correct null value -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// } - - doFinalizer(pCtx); -} - -/** - * 1. If the column value for filter exists, we need to load the SFields, which serves - * as the pre-filter to decide if the actual data block is required or not. - * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value. - * - * @param colId - * @param filterCols - * @return - */ -int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return BLK_DATA_NOT_LOAD; - } else { - return BLK_DATA_SMA_LOAD; - } -} - -int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_NOT_LOAD; -} -#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i])); \ - (numOfElem)++; \ - } \ - } while(0) -#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - SET_DOUBLE_VAL(&(x) , (x) + (d)[i]); \ - (numOfElem)++; \ - } \ - } while(0) - -#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType) \ - do { \ - t *d = (t *)(p); \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \ - continue; \ - }; \ - (x) += (d)[i]; \ - (numOfElem)++; \ - } \ - } while(0) - -#define UPDATE_DATA(ctx, left, right, num, sign, k) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS(ctx, k); \ - (num) += 1; \ - } \ - } while (0) - -#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \ - do { \ - if (((left) < (right)) ^ (sign)) { \ - (left) = (right); \ - DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx); \ - (num) += 1; \ - } \ - } while (0) - -#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num) \ - for (int32_t i = 0; i < ((ctx)->size); ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ - continue; \ - } \ - TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ - UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ - } - -#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \ - do { \ - type *_data = (type *)data; \ - type *_list = (type *)list; \ - LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ - } while (0) - -static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_SMA_LOAD; -} - -static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - return BLK_DATA_DATA_LOAD; -} - -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // no result for first query, data block is required - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { - return BLK_DATA_DATA_LOAD; - } else { - return BLK_DATA_NOT_LOAD; - } -} - -static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order == TSDB_ORDER_DESC) { - return BLK_DATA_NOT_LOAD; - } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { // data in current block is not earlier than current result - return (pInfo->ts <= w->skey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return BLK_DATA_NOT_LOAD; -// } - - // not initialized yet, it is the first block, load it. - if (pCtx->pOutput == NULL) { - return BLK_DATA_DATA_LOAD; - } - - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is - // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); - if (pInfo->hasResult != DATA_SET_FLAG) { - return BLK_DATA_DATA_LOAD; - } else { - return (pInfo->ts > w->ekey) ? BLK_DATA_NOT_LOAD : BLK_DATA_DATA_LOAD; - } -} - -////////////////////////////////////////////////////////////////////////////////////////////// -/* - * The intermediate result of average is kept in the interResultBuf. - * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate - * result into output buffer. - */ -static void avg_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - // NOTE: keep the intermediate result into the interResultBuf - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - double *pVal = &pAvgInfo->sum; - - if (pCtx->isAggSet) { // Pre-aggregation - notNullElems = pCtx->size - pCtx->agg.numOfNull; - assert(notNullElems >= 0); - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += pCtx->agg.sum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - *pVal += (uint64_t) pCtx->agg.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->agg.sum)); - } - } else { - void *pData = GET_INPUT_DATA_LIST(pCtx); - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); - } - } - - if (!pCtx->hasNull) { - assert(notNullElems == pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - pAvgInfo->num += notNullElems; - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); - } -} - -static void avg_func_merge(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - double *sum = (double*) pCtx->pOutput; - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SAvgInfo *pInput = (SAvgInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - SET_DOUBLE_VAL(sum, *sum + pInput->sum); - - // keep the number of data into the temp buffer - *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; - } -} - -/* - * the average value is calculated in finalize routine, since current routine does not know the exact number of points - */ -static void avg_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo)); - } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY - assert(IS_NUMERIC_TYPE(pCtx->inputType)); - SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num); - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// - -static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MAX; - break; - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->pOutput = UINT8_MAX; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MAX; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = UINT16_MAX; - break; - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MAX; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = UINT32_MAX; - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MAX; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = UINT64_MAX; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX); - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; // not initialized since it has been initialized - } - - GET_TRUE_DATA_TYPE(); - - switch (type) { - case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->pOutput) = INT32_MIN; - break; - case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->pOutput) = -FLT_MAX; - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX); - break; - case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->pOutput) = INT64_MIN; - break; - case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->pOutput) = INT16_MIN; - break; - case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->pOutput) = 0; - break; - case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->pOutput) = INT8_MIN; - break; - case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->pOutput) = 0; - break; - default: - assert(0); -// qError("illegal data type:%d in min/max query", pCtx->inputType); - } - - return true; -} - -/* - * the output result of min/max function is the final output buffer, not the intermediate result buffer - */ -static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { - int32_t notNullElems = 0; -#if 0 - GET_TRUE_DATA_TYPE(); - assert(pCtx->stableQuery); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *input = GET_INPUT_DATA(pCtx, i); - if (input[bytes] != DATA_SET_FLAG) { - continue; - } - - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t v = GET_INT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t v = GET_INT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t v = GET_INT32_VAL(input); - if ((*(int32_t *)output < v) ^ isMin) { - *(int32_t *)output = v; - - for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; - aggFunc[FUNCTION_TAG].addInput(__ctx); - } - - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float v = GET_FLOAT_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double v = GET_DOUBLE_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t v = GET_INT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t v = GET_UINT8_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t v = GET_UINT16_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UINT: { - uint32_t v = GET_UINT32_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin); - break; - } - - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t v = GET_UINT64_VAL(input); - DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin); - break; - } - - default: - break; - } - } -#endif - - return notNullElems; -} - -static void min_func_merge(SqlFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); - - SET_VAL(pCtx, notNullElems, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void max_func_merge(SqlFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); - - SET_VAL(pCtx, numOfElem, 1); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (numOfElem > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ - continue; \ - } \ - (num) += 1; \ - (r) += TPOW2(((type *)d)[i] - (delta)); \ - } - -static void stddev_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pStd->stage == 0) { - pStd->stage++; - avg_finalizer(pCtx); - - pResInfo->initialized = true; // set it initialized to avoid re-initialization - - // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - - pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); - assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); - } - - if (pStd->stage == 0) { - // the first stage is to calculate average value - avg_function(pCtx); - } else if (pStd->num > 0) { - // the second stage to calculate standard deviation - // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round - double *retVal = &pStd->res; - double avg = pStd->avg; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - assert(0); -// qError("stddev function not support data type:%d", pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } -} - -static void stddev_finalizer(SqlFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////// -int32_t tsCompare(const void* p1, const void* p2) { - TSKEY k = *(TSKEY*)p1; - SResPair* pair = (SResPair*)p2; - - if (k == pair->key) { - return 0; - } else { - return k < pair->key? -1:1; - } -} - -////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // used to keep the timestamp for comparison -// pCtx->param[1].param.nType = 0; -// pCtx->param[1].param.i = 0; - - return true; -} - -// todo opt for null block -static void first_function(SqlFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // handle the null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - if (pCtx->ptsList != NULL) { - TSKEY k = GET_TS_DATA(pCtx, i); -// DO_UPDATE_TAG_COLUMNS(pCtx, k); - } - - SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); -// pInfo->hasResult = DATA_SET_FLAG; - pInfo->complete = true; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -/* - * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) - * to decide if the value is earlier than current intermediate result - */ -static void first_dist_function(SqlFunctionCtx *pCtx) { - /* - * do not to check data in the following cases: - * 1. data block that are not loaded - * 2. scan data files in desc order - */ - if (pCtx->order == TSDB_ORDER_DESC) { - return; - } - - int32_t notNullElems = 0; - - // find the first not null value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - first_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void first_dist_func_merge(SqlFunctionCtx *pCtx) { - assert(pCtx->stableQuery); - - char * pData = GET_INPUT_DATA_LIST(pCtx); - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - // The param[1] is used to keep the initial value of max ts value -// if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i > pInput->ts) { -// memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); -// pCtx->param[1].param.i = pInput->ts; -// pCtx->param[1].param.nType = pCtx->resDataInfo.type; -// -//// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); -// } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////////////// -/* - * last function: - * 1. since the last block may be all null value, so, we simply access the last block is not valid - * each block need to be checked. - * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at - * least one data in this block that is not null.(TODO opt for this case) - */ -static void last_function(SqlFunctionCtx *pCtx) { -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - if (pCtx->order == TSDB_ORDER_DESC) { - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - //pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed on this column - notNullElems++; - break; - } - } else { // ascending order - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) { - continue; - } - - TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0; - - char* buf = GET_ROWCELL_INTERBUF(pResInfo); -// if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { -// //pResInfo->hasResult = DATA_SET_FLAG; -// memcpy(pCtx->pOutput, data, pCtx->inputBytes); -// -// *(TSKEY*)buf = ts; -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); -// } - - notNullElems++; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); -} - -static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { - int64_t *timestamp = GET_TS_LIST(pCtx); - - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); - - if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { -#if defined(_DEBUG_VIEW) - qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); -#endif - - memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pInfo->hasResult = DATA_SET_FLAG; - pInfo->ts = timestamp[index]; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts); - } -} - -static void last_dist_function(SqlFunctionCtx *pCtx) { - /* - * 1. for scan data is not the required order - * 2. for data blocks that are not loaded, no need to check data - */ -// if (pCtx->order != pCtx->param[0].param.i) { -// return; -// } - - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - if (!pCtx->requireNull) { - continue; - } - } - - last_data_assign_impl(pCtx, data, i); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - notNullElems++; - break; - } - - SET_VAL(pCtx, notNullElems, 1); -} - -/* - * in the secondary merge(local reduce), the output is limited by the - * final output size, so the main difference between last_dist_func_merge and second_merge - * is: the output data format in computing - */ -static void last_dist_func_merge(SqlFunctionCtx *pCtx) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - - SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); - if (pInput->hasResult != DATA_SET_FLAG) { - return; - } - - /* - * param[1] used to keep the corresponding timestamp to decide if current result is - * the true last result - */ - if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i < pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].param.i = pInput->ts; - pCtx->param[1].param.nType = pCtx->resDataInfo.type; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } - - SET_VAL(pCtx, 1, 1); -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -////////////////////////////////////////////////////////////////////////////////// -/* - * NOTE: last_row does not use the interResultBuf to keep the result - */ -static void last_row_function(SqlFunctionCtx *pCtx) { - assert(pCtx->size >= 1); - char *pData = GET_INPUT_DATA_LIST(pCtx); - - // assign the last element in current data block - assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer in case of super table query - if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); - pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); - pInfo1->hasResult = DATA_SET_FLAG; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); - } else { - TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1); -// DO_UPDATE_TAG_COLUMNS(pCtx, ts); - } - - SET_VAL(pCtx, pCtx->size, 1); -} - -static void last_row_finalizer(SqlFunctionCtx *pCtx) { - // do nothing at the first stage - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags, - SSubsidiaryResInfo *pTagInfo, int16_t stage) { - dst->v.nType = type; - dst->v.i = *(int64_t *)val; - dst->timestamp = tsKey; - - int32_t size = 0; - if (stage == MERGE_STAGE) { -// memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); - } else { // the tags are dumped from the ctx tag fields -// for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { -// SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; -// if (ctx->functionId == FUNCTION_TS_DUMMY) { -// ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; -// ctx->tag.i = tsKey; -// } -// -// taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); -// size += pTagInfo->pTagCtxList[i]->resDataInfo.bytes; -// } - } -} - -#define VALUEPAIRASSIGN(dst, src, __l) \ - do { \ - (dst)->timestamp = (src)->timestamp; \ - (dst)->v = (src)->v; \ - memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ - } while (0) - -static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) -{ - uint16_t type = *(uint16_t *) param; - tValuePair *val1 = *(tValuePair **) p1; - tValuePair *val2 = *(tValuePair **) p2; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (val1->v.i == val2->v.i) { - return 0; - } - - return (val1->v.i > val2->v.i) ? 1 : -1; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (val1->v.u == val2->v.u) { - return 0; - } - - return (val1->v.u > val2->v.u) ? 1 : -1; - } - - if (val1->v.d == val2->v.d) { - return 0; - } - - return (val1->v.d > val2->v.d) ? 1 : -1; -} - -static void topBotSwapFn(void *dst, void *src, const void *param) -{ - char tag[32768]; - tValuePair temp; - uint16_t tagLen = *(uint16_t *) param; - tValuePair *vdst = *(tValuePair **) dst; - tValuePair *vsrc = *(tValuePair **) src; - - memset(tag, 0, sizeof(tag)); - temp.pTags = tag; - - VALUEPAIRASSIGN(&temp, vdst, tagLen); - VALUEPAIRASSIGN(vdst, vsrc, tagLen); - VALUEPAIRASSIGN(vsrc, &temp, tagLen); -} - -static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); - } - } -} - -static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, - SSubsidiaryResInfo *pTagInfo, char *pTags, int16_t stage) { - SVariant val = {0}; - taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); - - tValuePair **pList = pInfo->res; - assert(pList != NULL); - - if (pInfo->num < maxLen) { - valuePairAssign(pList[pInfo->num], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); - -// taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - - pInfo->num++; - } else { - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i < pList[0]->v.i) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u < pList[0]->v.u) || - (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) { - valuePairAssign(pList[0], type, (const char *)&val.i, ts, pTags, pTagInfo, stage); -// taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); - } - } -} - -static int32_t resAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (pLeftElem->timestamp == pRightElem->timestamp) { - return 0; - } else { - return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1; - } -} - -static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); } - -static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { - tValuePair *pLeftElem = *(tValuePair **)pLeft; - tValuePair *pRightElem = *(tValuePair **)pRight; - - if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { - if (pLeftElem->v.d == pRightElem->v.d) { - return 0; - } else { - return pLeftElem->v.d > pRightElem->v.d ? 1 : -1; - } - } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ - if (pLeftElem->v.i == pRightElem->v.i) { - return 0; - } else { - return pLeftElem->v.i > pRightElem->v.i ? 1 : -1; - } - } else { - if (pLeftElem->v.u == pRightElem->v.u) { - return 0; - } else { - return pLeftElem->v.u > pRightElem->v.u ? 1 : -1; - } - } -} - -static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } - -static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - - tValuePair **tvp = pRes->res; - - int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); - - switch (type) { - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int32_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - SET_DOUBLE_VAL(output, tvp[i]->v.d); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (float)tvp[i]->v.d; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int16_t)tvp[i]->v.i; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->pOutput; - for (int32_t i = 0; i < len; ++i, output += step) { - *output = (int8_t)tvp[i]->v.i; - } - break; - } - default: { -// qError("top/bottom function not support data type:%d", pCtx->inputType); - return; - } - } - - // set the output timestamp of each record. -// TSKEY *output = pCtx->pTsOutput; -// for (int32_t i = 0; i < len; ++i, output += step) { -// *output = tvp[i]->timestamp; -// } - - // set the corresponding tag data for each record - // todo check malloc failure -// char **pData = taosMemoryCalloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); -// for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { -// pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; -// } - -// for (int32_t i = 0; i < len; ++i, output += step) { -// int16_t offset = 0; -// for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { -// memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes); -// offset += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// pData[j] += pCtx->tagInfo.pTagCtxList[j]->resDataInfo.bytes; -// } -// } - -// taosMemoryFreeClear(pData); -} - -/* - * Parameters values: - * 1. param[0]: maximum allowable results - * 2. param[1]: order by type (time or value) - * 3. param[2]: asc/desc order - * - * top/bottom use the intermediate result buffer to keep the intermediate result - */ -static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // only the first_stage_merge is directly written data into final output buffer - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - return (STopBotInfo*) pCtx->pOutput; - } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return GET_ROWCELL_INTERBUF(pResInfo); - } -} - - -/* - * keep the intermediate results during scan data blocks in the format of: - * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+ - * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| - * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ - */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { - char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = (tValuePair**) tmp; -// tmp += POINTER_BYTES * pCtx->param[0].param.i; - -// size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; - -// for (int32_t i = 0; i < pCtx->param[0].param.i; ++i) { -// pTopBotInfo->res[i] = (tValuePair*) tmp; -// pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); -// tmp += size; -// } -} - -bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo == NULL) { - return true; - } - - STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); - - // required number of results are not reached, continue load data block -// if (pTopBotInfo->num < pCtx->param[0].param.i) { -// return true; -// } - -// if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pTopBotInfo, pCtx); -// } - - tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - - if (pCtx->functionId == FUNCTION_TOP) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(maxval) > pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(maxval) > pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(maxval) > pRes[0]->v.d; - default: - return true; - } - } else { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: - return GET_INT8_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_SMALLINT: - return GET_INT16_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_INT: - return GET_INT32_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_BIGINT: - return GET_INT64_VAL(minval) < pRes[0]->v.i; - case TSDB_DATA_TYPE_FLOAT: - return GET_FLOAT_VAL(minval) < pRes[0]->v.d; - case TSDB_DATA_TYPE_DOUBLE: - return GET_DOUBLE_VAL(minval) < pRes[0]->v.d; - default: - return true; - } - } -} - -static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STopBotInfo *pInfo = getTopBotOutputInfo(pCtx); - buildTopBotStruct(pInfo, pCtx); - return true; -} - -static void top_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - assert(pRes->num >= 0); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_top_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; -// do_top_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, -// type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - -// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { -// buildTopBotStruct(pRes, pCtx); -// } - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems++; - // NOTE: Set the default timestamp if it is missing [todo refactor] - TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - // treat the result as only one result - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void bottom_func_merge(SqlFunctionCtx *pCtx) { - STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); - - // construct the input data struct from binary data - buildTopBotStruct(pInput, pCtx); - - STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); - - // the intermediate result is binary, we only use the output data type - for (int32_t i = 0; i < pInput->num; ++i) { - int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; -// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, -// &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->scanFlag); - } - - SET_VAL(pCtx, pInput->num, pOutput->num); - - if (pOutput->num > 0) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); - if (pRes->num == 0) { // no result -// assert(pResInfo->hasResult != DATA_SET_FLAG); - // TODO: - } - - GET_RES_INFO(pCtx)->numOfRes = pRes->num; - tValuePair **tvp = pRes->res; - - // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { - __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } - - GET_TRUE_DATA_TYPE(); - copyTopBotRes(pCtx, type); - - doFinalizer(pCtx); -} - -/////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // in the first round, get the min-max value of all involved data - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo); - SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); - SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); - pInfo->numOfElems = 0; - - return true; -} - -static void percentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == REPEAT_SCAN && pInfo->stage == 0) { - pInfo->stage += 1; - - // all data are null, set it completed - if (pInfo->numOfElems == 0) { - pResInfo->complete = true; - - return; - } else { - pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); - } - } - - // the first stage, only acquire the min/max value - if (pInfo->stage == 0) { - if (pCtx->isAggSet) { - double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_INT64_VAL(&pCtx->agg.min); - tmax = (double)GET_INT64_VAL(&pCtx->agg.max); - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - tmin = GET_DOUBLE_VAL(&pCtx->agg.min); - tmax = GET_DOUBLE_VAL(&pCtx->agg.max); - } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - tmin = (double)GET_UINT64_VAL(&pCtx->agg.min); - tmax = (double)GET_UINT64_VAL(&pCtx->agg.max); - } else { - assert(true); - } - - if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { - SET_DOUBLE_VAL(&pInfo->minval, tmin); - } - - if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) { - SET_DOUBLE_VAL(&pInfo->maxval, tmax); - } - - pInfo->numOfElems += (pCtx->size - pCtx->agg.numOfNull); - } else { - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - - if (v < GET_DOUBLE_VAL(&pInfo->minval)) { - SET_DOUBLE_VAL(&pInfo->minval, v); - } - - if (v > GET_DOUBLE_VAL(&pInfo->maxval)) { - SET_DOUBLE_VAL(&pInfo->maxval, v); - } - - pInfo->numOfElems += 1; - } - } - - return; - } - - // the second stage, calculate the true percentile value - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - tMemBucketPut(pInfo->pMemBucket, data, 1); - } - - SET_VAL(pCtx, notNullElems, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void percentile_finalizer(SqlFunctionCtx *pCtx) { -// double v = pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); - - tMemBucket * pMemBucket = ppInfo->pMemBucket; - if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null - assert(ppInfo->numOfElems == 0); - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { -// SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); - } - - tMemBucketDestroy(pMemBucket); - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////// -static void buildHistogramInfo(SAPercentileInfo* pInfo) { - pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo)); - pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); -} - -static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo* pInfo = NULL; - - if (pCtx->stableQuery && pCtx->scanFlag != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->pOutput; - } else { - pInfo = GET_ROWCELL_INTERBUF(pResInfo); - } - - buildHistogramInfo(pInfo); - return pInfo; -} - -static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - char *tmp = (char *)pInfo + sizeof(SAPercentileInfo); - pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN); - return true; -} - -static void apercentile_function(SqlFunctionCtx *pCtx) { - int32_t notNullElems = 0; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); - - assert(pInfo->pHisto->elems != NULL); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *data = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(data, pCtx->inputType)) { - continue; - } - - notNullElems += 1; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, data); - tHistogramAdd(&pInfo->pHisto, v); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } -} - -static void apercentile_func_merge(SqlFunctionCtx *pCtx) { - SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); - - pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); - pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); - - if (pInput->pHisto->numOfElems <= 0) { - return; - } - - SAPercentileInfo *pOutput = getAPerctInfo(pCtx); - SHistogramInfo *pHisto = pOutput->pHisto; - - if (pHisto->numOfElems <= 0) { - memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - } else { - //TODO(dengyihao): avoid memcpy - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); - memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); - tHistogramDestroy(&pRes); - } - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - //pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, 1, 1); -} - -static void apercentile_finalizer(SqlFunctionCtx *pCtx) { - double v = (pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].param.i : pCtx->param[0].param.d; - - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); - - if (pCtx->scanFlag == MERGE_STAGE) { -// if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null -// assert(pOutput->pHisto->numOfElems > 0); -// -// double ratio[] = {v}; -// double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); -// -// memcpy(pCtx->pOutput, res, sizeof(double)); -// taosMemoryFree(res); -// } else { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - } else { - if (pOutput->pHisto->numOfElems > 0) { - double ratio[] = {v}; - - double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->pOutput, res, sizeof(double)); - taosMemoryFree(res); - } else { // no need to free - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - } - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // 2*3 matrix -// pInfo->startVal = pCtx->param[0].param.d; - return true; -} - -#define LEASTSQR_CAL(p, x, y, index, step) \ - do { \ - (p)[0][0] += (double)(x) * (x); \ - (p)[0][1] += (double)(x); \ - (p)[0][2] += (double)(x) * (y)[index]; \ - (p)[1][2] += (y)[index]; \ - (x) += step; \ - } while (0) - -#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \ - continue; \ - } \ - (n)++; \ - LEASTSQR_CAL(param, x, y, i, step); \ - } - -static void leastsquares_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - double(*param)[3] = pInfo->mat; - double x = pInfo->startVal; - - void *pData = GET_INPUT_DATA_LIST(pCtx); - - int32_t numOfElem = 0; - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *p = pData; - // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { - continue; - } - - param[0][0] += x * x; - param[0][1] += x; - param[0][2] += x * p[i]; - param[1][2] += p[i]; - - x += pCtx->param[1].param.d; - numOfElem++; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - }; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); - break; - } - } - - pInfo->startVal = x; - pInfo->num += numOfElem; - - if (pInfo->num > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - SET_VAL(pCtx, numOfElem, 1); -} - -static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { - // no data in query - SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->num == 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - double(*param)[3] = pInfo->mat; - - param[1][1] = (double)pInfo->num; - param[1][0] = param[0][1]; - - param[0][0] -= param[1][0] * (param[0][1] / param[1][1]); - param[0][2] -= param[1][2] * (param[0][1] / param[1][1]); - param[0][1] = 0; - param[1][2] -= param[0][2] * (param[1][0] / param[0][0]); - param[1][0] = 0; - param[0][2] /= param[0][0]; - - param[1][2] /= param[1][1]; - - int32_t maxOutputSize = AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", - param[0][2], param[1][2]); - - varDataSetLen(pCtx->pOutput, n); - doFinalizer(pCtx); -} - -static void date_col_output_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->pOutput) = pCtx->startTs; -} - -static void col_project_function(SqlFunctionCtx *pCtx) { - // the number of output rows should not affect the final number of rows, so set it to be 0 - if (pCtx->numOfParams == 2) { - return; - } - - // only one row is required. -// if (pCtx->param[0].param.i == 1) { -// SET_VAL(pCtx, pCtx->size, 1); -// } else { -// INC_INIT_VAL(pCtx, pCtx->size); -// } - - char *pData = GET_INPUT_DATA_LIST(pCtx); - if (pCtx->order == TSDB_ORDER_ASC) { -// int32_t numOfRows = (pCtx->param[0].param.i == 1)? 1:pCtx->size; -// memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); - } else { - for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, - pCtx->inputBytes); - } - } -} - -/** - * only used for tag projection query in select clause - * @param pCtx - * @return - */ -static void tag_project_function(SqlFunctionCtx *pCtx) { - INC_INIT_VAL(pCtx, pCtx->size); - - assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); - - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - char* data = pCtx->pOutput; - pCtx->pOutput += pCtx->resDataInfo.bytes; - - // directly copy from the first one - for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->pOutput, data, pCtx->resDataInfo.bytes); - pCtx->pOutput += pCtx->resDataInfo.bytes; - } -} - -/** - * used in group by clause. when applying group by tags, the tags value is - * assign by using tag function. - * NOTE: there is only ONE output for ONE query range - * @param pCtx - * @return - */ -static void copy_function(SqlFunctionCtx *pCtx); - -static void tag_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, 1, 1); - if (pCtx->scanFlag == MERGE_STAGE) { - copy_function(pCtx); - } else { - taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->resDataInfo.type, true); - } -} - -static void copy_function(SqlFunctionCtx *pCtx) { - SET_VAL(pCtx, pCtx->size, 1); - - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); -} - -enum { - INITIAL_VALUE_NOT_ASSIGNED = 0, -}; - -static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - // diff function require the value is set to -1 - pCtx->param[1].param.nType = INITIAL_VALUE_NOT_ASSIGNED; - return false; -} - -static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { - if (!function_setup(pCtx, pResultInfo)) { - return false; - } - - // diff function require the value is set to -1 - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - -// pDerivInfo->ignoreNegative = pCtx->param[1].param.i; - pDerivInfo->prevTs = -1; -// pDerivInfo->tsWindow = pCtx->param[0].param.i; - pDerivInfo->valueSet = false; - return false; -} - -static void deriv_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); - - void *data = GET_INPUT_DATA_LIST(pCtx); - - int32_t notNullElems = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY *pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY *tsList = GET_TS_LIST(pCtx); - - double *pOutput = (double *)pCtx->pOutput; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs)); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - - break; - }; - - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = (double) pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (!pDerivInfo->valueSet) { // initial value is not set yet - pDerivInfo->valueSet = true; - } else { - *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs); - if (pDerivInfo->ignoreNegative && *pOutput < 0) { - } else { - *pTimestamp = tsList[i]; - - pOutput += 1; - pTimestamp += 1; - notNullElems++; - } - } - - pDerivInfo->prevValue = pData[i]; - pDerivInfo->prevTs = tsList[i]; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - GET_RES_INFO(pCtx)->numOfRes += notNullElems; -} - -// TODO difference in date column -static void diff_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED); - - int32_t notNullElems = 0; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1; - - TSKEY* pTimestamp = NULL;//pCtx->pTsOutput; - TSKEY* tsList = GET_TS_LIST(pCtx); - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - }; - case TSDB_DATA_TYPE_BIGINT: { - int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].param.i; // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *pData = (double *)data; - double *pOutput = (double *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *pData = (float *)data; - float *pOutput = (float *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].param.d); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.d = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->pOutput; - - for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { - continue; - } - - if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null - *pTimestamp = (tsList != NULL)? tsList[i]:0; - pOutput += 1; - pTimestamp += 1; - } - - pCtx->param[1].param.i = pData[i]; - pCtx->param[1].param.nType = pCtx->inputType; - notNullElems++; - } - break; - } - default: - assert(0); -// qError("error input type"); - } - - // initial value is not set yet - if (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { - /* - * 1. current block and blocks before are full of null - * 2. current block may be null value - */ - assert(pCtx->hasNull); - } else { - int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; - - GET_RES_INFO(pCtx)->numOfRes += forwardStep; - } -} - -#if 0 -char *getArithColumnData(void *param, const char* name, int32_t colId) { - SScalarFunctionSupport *pSupport = (SScalarFunctionSupport *)param; - - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pSupport->colList[i].colId) { - index = i; - break; - } - } - - assert(index >= 0); - return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; -} -#endif - -static void arithmetic_function(SqlFunctionCtx *pCtx) { - GET_RES_INFO(pCtx)->numOfRes += pCtx->size; - //SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; - -// SScalarParam output = {0}; -// output.data = pCtx->pOutput; - - //evaluateExprNodeTree(pSup->pExprInfo->pExpr, pCtx->size, &output, pSup, getArithColumnData); -} - -#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ - { \ - type *inputData = (type *)data; \ - for (int32_t i = 0; i < elemCnt; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) { \ - continue; \ - } \ - if (inputData[i] < minOutput) { \ - minOutput = (double)inputData[i]; \ - } \ - if (inputData[i] > maxOutput) { \ - maxOutput = (double)inputData[i]; \ - } \ - numOfNotNullElem++; \ - } \ - } - -///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // this is the server-side setup function in client-side, the secondary merge do not need this procedure - if (pCtx->scanFlag == MERGE_STAGE) { -// pCtx->param[0].param.d = DBL_MAX; -// pCtx->param[3].param.d = -DBL_MAX; - } else { - pInfo->min = DBL_MAX; - pInfo->max = -DBL_MAX; - } - - return true; -} - -static void spread_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - int32_t numOfElems = 0; - - // todo : opt with pre-calculated result - // column missing cause the hasNull to be true - if (pCtx->isAggSet) { - numOfElems = pCtx->size - pCtx->agg.numOfNull; - - // all data are null in current data block, ignore current data block - if (numOfElems == 0) { - goto _spread_over; - } - - if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) || - (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) { - if (pInfo->min > pCtx->agg.min) { - pInfo->min = (double)pCtx->agg.min; - } - - if (pInfo->max < pCtx->agg.max) { - pInfo->max = (double)pCtx->agg.max; - } - } else if (IS_FLOAT_TYPE(pCtx->inputType)) { - if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->agg.min))) { - pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->agg.min)); - } - - if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->agg.max))) { - pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->agg.max)); - } - } - - goto _spread_over; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - numOfElems = 0; - - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { - LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); - } - - if (!pCtx->hasNull) { - assert(pCtx->size == numOfElems); - } - - _spread_over: - SET_VAL(pCtx, numOfElems, 1); - - if (numOfElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - pInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); - } -} - -/* - * here we set the result value back to the intermediate buffer, to apply the finalize the function - * the final result is generated in spread_function_finalizer - */ -void spread_func_merge(SqlFunctionCtx *pCtx) { - SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); - if (pData->hasResult != DATA_SET_FLAG) { - return; - } - -// if (pCtx->param[0].param.d > pData->min) { -// pCtx->param[0].param.d = pData->min; -// } - -// if (pCtx->param[3].param.d < pData->max) { -// pCtx->param[3].param.d = pData->max; -// } - -// GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; -} - -void spread_function_finalizer(SqlFunctionCtx *pCtx) { - /* - * here we do not check the input data types, because in case of metric query, - * the type of intermediate data is binary - */ - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - if (pCtx->scanFlag == MERGE_STAGE) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - -// if (pResInfo->hasResult != DATA_SET_FLAG) { -// setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); -// return; -// } - -// SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].param.d - pCtx->param[0].param.d); - } else { - assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - - SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - return; - } - - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case - doFinalizer(pCtx); -} - - -/** - * param[1]: start time - * param[2]: end time - * @param pCtx - */ -static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->p.key = INT64_MIN; - pInfo->win = TSWINDOW_INITIALIZER; - return true; -} - -static double twa_get_area(SPoint1 s, SPoint1 e) { - if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { - return (s.val + e.val) * (e.key - s.key) / 2; - } - - double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); - double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; - return val; -} - -static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { - int32_t notNullElems = 0; - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *tsList = GET_TS_LIST(pCtx); - - int32_t i = index; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - SPoint1* last = &pInfo->p; - - if (pCtx->start.key != INT64_MIN) { - assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); - - assert(last->key == INT64_MIN); - - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; - notNullElems++; - i += step; - } else if (pInfo->p.key == INT64_MIN) { - last->key = tsList[i]; - GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index)); - - pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = last->key; - notNullElems++; - i += step; - } - - // calculate the value of - switch(pCtx->inputType) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); - for (; i < size && i >= 0; i += step) { - if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { - continue; - } - -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif - pInfo->dOutput += twa_get_area(pInfo->p, st); - pInfo->p = st; - } - break; - } - default: assert(0); - } - - // the last interpolated time window value - if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); - pInfo->p = pCtx->end; - } - - pInfo->win.ekey = pInfo->p.key; - return notNullElems; -} - -static void twa_function(SqlFunctionCtx *pCtx) { - void *data = GET_INPUT_DATA_LIST(pCtx); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - // skip null value - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); - while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i += step; - } - - int32_t notNullElems = 0; - if (i >= 0 && i < pCtx->size) { - notNullElems = twa_function_impl(pCtx, i, pCtx->size); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - //pResInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); - } -} - -/* - * To copy the input to interResBuf to avoid the input buffer space be over writen - * by next input data. The TWA function only applies to each table, so no merge procedure - * is required, we simply copy to the resut ot interResBuffer. - */ -void twa_function_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; -} - -void twa_function_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); - if (pInfo->win.ekey == pInfo->win.skey) { - SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val); - } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey)); - } - - GET_RES_INFO(pCtx)->numOfRes = 1; - doFinalizer(pCtx); -} - -/** - * - * @param pCtx - */ - -static void interp_function_impl(SqlFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].param.i; - if (type == TSDB_FILL_NONE) { - return; - } - - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - - if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { - *(TSKEY *)pCtx->pOutput = pCtx->startTs; - } else if (type == TSDB_FILL_NULL) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else if (type == TSDB_FILL_SET_VALUE) { -// taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); - } else { - if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { - if (type == TSDB_FILL_PREV) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); - } else { - assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_NEXT) { - if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); - } else { - assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->resDataInfo.bytes, pCtx->inputType); - } - } else if (type == TSDB_FILL_LINEAR) { - SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; - SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } else { - // no data generated yet - if (pCtx->size < 1) { - return; - } - - // check the timestamp in input buffer - TSKEY skey = GET_TS_DATA(pCtx, 0); - - if (type == TSDB_FILL_PREV) { - if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) { - return; - } - - if (pCtx->size > 1) { - TSKEY ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey > skey && ekey <= pCtx->startTs) || - ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){ - skey = ekey; - } - } -// assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_NEXT) { - TSKEY ekey = skey; - char* val = NULL; - - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - if (pCtx->size > 1) { - ekey = GET_TS_DATA(pCtx, 1); - if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) { - return; - } - - val = ((char*)pCtx->pInput) + pCtx->inputBytes; - } else { - return; - } - } else { - val = (char*)pCtx->pInput; - } - - assignVal(pCtx->pOutput, val, pCtx->resDataInfo.bytes, pCtx->inputType); - } else if (type == TSDB_FILL_LINEAR) { - if (pCtx->size <= 1) { - return; - } - - TSKEY ekey = GET_TS_DATA(pCtx, 1); - - // no data generated yet - if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs)) - || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) { - return; - } - - char *start = GET_INPUT_DATA(pCtx, 0); - char *end = GET_INPUT_DATA(pCtx, 1); - - SPoint point1 = {.key = skey, .val = start}; - SPoint point2 = {.key = ekey, .val = end}; - SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; - - int32_t srcType = pCtx->inputType; - if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? - if (isNull(start, srcType) || isNull(end, srcType)) { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } else { -// taosGetLinearInterpolationVal(&point, pCtx->resDataInfo.type, &point1, &point2, srcType); - } - } else { - setNull(pCtx->pOutput, srcType, pCtx->inputBytes); - } - } - } - } - - SET_VAL(pCtx, 1, 1); -} - -static void interp_function(SqlFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - if (pCtx->size > 0) { - bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); - TSKEY key; - char *pData; - int32_t typedData = 0; - - if (ascQuery) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - key = pCtx->start.key; - if (key == INT64_MIN) { - key = GET_TS_DATA(pCtx, 0); - pData = GET_INPUT_DATA(pCtx, 0); - } else { - if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) { - pData = pCtx->start.ptr; - } else { - typedData = 1; - pData = (char *)&pCtx->start.val; - } - } - } - - //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) { - if (key == pCtx->startTs) { - if (typedData) { - SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData); - } else { - assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); - } - - SET_VAL(pCtx, 1, 1); - } else { - interp_function_impl(pCtx); - } - } else { //no qualified data rows and interpolation is required - interp_function_impl(pCtx); - } -} - -static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; // not initialized since it has been initialized - } - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->pTSBuf = tsBufCreate(false, pCtx->order); - pInfo->pTSBuf->tsOrder = pCtx->order; - return true; -} - -static void ts_comp_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; - - const char *input = GET_INPUT_DATA_LIST(pCtx); - - // primary ts must be existed, so no need to check its existance - if (pCtx->order == TSDB_ORDER_ASC) { -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); - } else { - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *d = GET_INPUT_DATA(pCtx, i); -// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); - } - } - - SET_VAL(pCtx, pCtx->size, 1); - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void ts_comp_finalize(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - STSBuf * pTSbuf = pInfo->pTSBuf; - - tsBufFlush(pTSbuf); -// qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); - - // TODO refactor transfer ownership of current file - *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; - - pResInfo->complete = true; - - // get the file size - int64_t file_size; - if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { - pResInfo->numOfRes = (uint32_t )file_size; - } - - pTSbuf->remainOpen = true; - tsBufDestroy(pTSbuf); - - doFinalizer(pCtx); -} - -////////////////////////////////////////////////////////////////////////////////////////////// -// rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || - (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0.0; - } - - double diff = 0; - if (pRateInfo->isIRate) { - // If the previous value of the last is greater than the last value, only keep the last point instead of the delta - // value between two values. - diff = pRateInfo->lastValue; - if (diff >= pRateInfo->firstValue) { - diff -= pRateInfo->firstValue; - } - } else { - diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; - if (diff <= 0) { - return 0; - } - } - - int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - if (duration == 0) { - return 0; - } - - return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; -} - -static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { - if (!function_setup(pCtx, pResInfo)) { - return false; - } - - SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->correctionValue = 0; - pInfo->firstKey = INT64_MIN; - pInfo->lastKey = INT64_MIN; - pInfo->firstValue = (double) INT64_MIN; - pInfo->lastValue = (double) INT64_MIN; - - pInfo->hasResult = 0; - pInfo->isIRate = (pCtx->functionId == FUNCTION_IRATE); - return true; -} - -static void rate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - -// qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - for (int32_t i = 0; i < pCtx->size; ++i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { -// qDebug("%p rate_function() index of null data:%d", pCtx, i); - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - } - - if (INT64_MIN == pRateInfo->lastValue) { - pRateInfo->lastValue = v; - } else if (v < pRateInfo->lastValue) { - pRateInfo->correctionValue += pRateInfo->lastValue; - } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - } - - if (!pCtx->hasNull) { - assert(pCtx->size == notNullElems); - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void rate_func_copy(SqlFunctionCtx *pCtx) { - assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); -// pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; -} - -static void rate_finalizer(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - -// SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].param.i))); - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -static void irate_function(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - - int32_t notNullElems = 0; - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); - - for (int32_t i = pCtx->size - 1; i >= 0; --i) { - char *pData = GET_INPUT_DATA(pCtx, i); - if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - continue; - } - - notNullElems++; - - double v = 0; - GET_TYPED_DATA(v, double, pCtx->inputType, pData); - - if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = primaryKey[i]; - continue; - } - - if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = primaryKey[i]; - break; - } - } - - SET_VAL(pCtx, notNullElems, 1); - - if (notNullElems > 0) { - pRateInfo->hasResult = DATA_SET_FLAG; -// pResInfo->hasResult = DATA_SET_FLAG; - } - - // keep the data into the final output buffer for super table query since this execution may be the last one - if (pCtx->stableQuery) { - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDistInfo* pDist) { - SBufferReader br = tbufInitReader(data, len, false); - - pDist->numOfTables = tbufReadUint32(&br); - pDist->numOfFiles = tbufReadUint16(&br); - pDist->totalSize = tbufReadUint64(&br); - pDist->totalRows = tbufReadUint64(&br); - pDist->maxRows = tbufReadInt32(&br); - pDist->minRows = tbufReadInt32(&br); - pDist->numOfInmemRows = tbufReadUint32(&br); - pDist->numOfSmallBlocks = tbufReadUint32(&br); - int64_t numSteps = tbufReadUint64(&br); - - bool comp = tbufReadUint8(&br); - uint32_t compLen = tbufReadUint32(&br); - - size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo)); - - char* outputBuf = NULL; - if (comp) { - outputBuf = taosMemoryMalloc(originalLen); - - size_t actualLen = compLen; - const char* compStr = tbufReadBinary(&br, &actualLen); - - int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, - (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); - assert(orignalLen == numSteps *sizeof(SFileBlockInfo)); - } else { - outputBuf = (char*) tbufReadBinary(&br, &originalLen); - } - - pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo)); - if (comp) { - taosMemoryFreeClear(outputBuf); - } -} - -static void blockInfo_func(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - - memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlockDistInfo* pSrc) { - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - assert(pDist != NULL && pSrc != NULL); - - pDist->numOfTables += pSrc->numOfTables; - pDist->numOfInmemRows += pSrc->numOfInmemRows; - pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks; - pDist->numOfFiles += pSrc->numOfFiles; - pDist->totalSize += pSrc->totalSize; - pDist->totalRows += pSrc->totalRows; - -// if (pResInfo->hasResult == DATA_SET_FLAG) { -// pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows); -// pDist->minRows = TMIN(pDist->minRows, pSrc->minRows); -// } else { - pDist->maxRows = pSrc->maxRows; - pDist->minRows = pSrc->minRows; - - int32_t maxSteps = TSDB_MAX_MAXROWS_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS; - if (TSDB_MAX_MAXROWS_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) { - ++maxSteps; - } - pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo)); - taosArraySetSize(pDist->dataBlockInfos, maxSteps); -// } - - size_t steps = taosArrayGetSize(pSrc->dataBlockInfos); - for (int32_t i = 0; i < steps; ++i) { - int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep; - SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i); - blockInfo->numBlocksOfStep += srcNumBlocks; - } -} - -void block_func_merge(SqlFunctionCtx* pCtx) { - STableBlockDistInfo info = {0}; - int32_t len = *(int32_t*) pCtx->pInput; - blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - mergeTableBlockDist(pResInfo, &info); - taosArrayDestroy(info.dataBlockInfos); - - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; -} - -void getPercentiles(STableBlockDistInfo *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents, - double* percents, int32_t* percentiles) { - if (totalBlocks == 0) { - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = 0; - } - return; - } - - SArray *blocksInfos = pTableBlockDist->dataBlockInfos; - size_t numSteps = taosArrayGetSize(blocksInfos); - size_t cumulativeBlocks = 0; - - int percentIndex = 0; - for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) { - int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep; - if (numStepBlocks == 0) continue; - cumulativeBlocks += numStepBlocks; - - while (percentIndex < numOfPercents) { - double blockRank = totalBlocks * percents[percentIndex]; - if (blockRank <= cumulativeBlocks) { - percentiles[percentIndex] = indexStep; - ++percentIndex; - } else { - break; - } - } - } - - for (int32_t i = 0; i < numOfPercents; ++i) { - percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2; - } -} - -void generateBlockDistResult(STableBlockDistInfo *pTableBlockDist, char* result) { - if (pTableBlockDist == NULL) { - return; - } - - SArray* blockInfos = pTableBlockDist->dataBlockInfos; - uint64_t totalRows = pTableBlockDist->totalRows; - size_t numSteps = taosArrayGetSize(blockInfos); - int64_t totalBlocks = 0; - int64_t min = -1, max = -1, avg = 0; - - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - totalBlocks += blocks; - } - - avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; - min = totalBlocks > 0 ? pTableBlockDist->minRows : 0; - max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0; - - double stdDev = 0; - if (totalBlocks > 0) { - double variance = 0; - for (int32_t i = 0; i < numSteps; i++) { - SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); - int64_t blocks = blockInfo->numBlocksOfStep; - int32_t rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2); - variance += blocks * (rows - avg) * (rows - avg); - } - variance = variance / totalBlocks; - stdDev = sqrt(variance); - } - - double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99}; - int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; - assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t)); - getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles); - - uint64_t totalLen = pTableBlockDist->totalSize; - int32_t rowSize = pTableBlockDist->rowSize; - int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks; - double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1; - int sz = sprintf(result + VARSTR_HEADER_SIZE, - "summary: \n\t " - "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " - "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " - "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " - "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t " - "RowsInMem=[%d] \n\t", - percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5], - percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11], - min, max, avg, stdDev, - totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio, - pTableBlockDist->numOfInmemRows); - varDataSetLen(result, sz); - UNUSED(sz); -} - -void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - -// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; - generateBlockDistResult(pDist, pCtx->pOutput); - - if (pDist->dataBlockInfos != NULL) { - taosArrayDestroy(pDist->dataBlockInfos); - pDist->dataBlockInfos = NULL; - } - - // cannot set the numOfIteratedElems again since it is set during previous iteration - pResInfo->numOfRes = 1; - //pResInfo->hasResult = DATA_SET_FLAG; - - doFinalizer(pCtx); -} - -///////////////////////////////////////////////////////////////////////////////////////////// -/* - * function compatible list. - * tag and ts are not involved in the compatibility check - * - * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom - * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp - * 3. functions that are allowed to be present with other functions. - * e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last... - * - */ -int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp - 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, - // tid_tag, derivative, blk_info - 6, 8, 7, -}; -#endif diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c deleted file mode 100644 index 039e0a9dfc..0000000000 --- a/source/libs/function/src/texpr.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "function.h" -#include "os.h" - -#include "texception.h" -#include "tmsg.h" - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); - -void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { - if (pNode == NULL) { - return; - } - - if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) { - doExprTreeDestroy(&pNode, fp); - } - taosMemoryFree(pNode); -} - -static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { - if (*pExpr == NULL) { - return; - } - taosMemoryFree(*pExpr); - *pExpr = NULL; -} - -// TODO: these three functions should be made global -static void* exception_calloc(size_t nmemb, size_t size) { - void* p = taosMemoryCalloc(nmemb, size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static void* exception_malloc(size_t size) { - void* p = taosMemoryMalloc(size); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - -static UNUSED_FUNC char* exception_strdup(const char* str) { - char* p = strdup(str); - if (p == NULL) { - THROW(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - return p; -} - diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c new file mode 100644 index 0000000000..ff15b7714f --- /dev/null +++ b/source/libs/function/src/tfunctionInt.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taosdef.h" +#include "tmsg.h" +#include "thash.h" +#include "ttypes.h" + +#include "function.h" +#include "tbuffer.h" +#include "tcompression.h" +#include "tdatablock.h" +#include "tfunctionInt.h" +#include "thistogram.h" +#include "tpercentile.h" +#include "ttszip.h" +#include "tudf.h" + +void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { + pCell->initialized = false; +} + +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { + int32_t maxRows = 0; + + for (int32_t j = 0; j < num; ++j) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { + maxRows = pResInfo->numOfRes; + } + } + + assert(maxRows >= 0); + + blockDataEnsureCapacity(pResBlock, maxRows); + for(int32_t i = 0; i < num; ++i) { + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[i]); + if (pResInfo->numOfRes == 0) { + for(int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, NULL, true); // TODO add set null data api + } + } else { + for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { + colDataAppend(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); + } + } + } + + pResBlock->info.rows = maxRows; + return maxRows; +} + +bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { + assert(pEntry != NULL); + return pEntry->complete; +} + +bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { + return pEntry->initialized; +} diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 9c9807f843..b7532173c8 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1192,7 +1192,10 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, pBuilder->hasNone = true; } + tdSRowEnd(pBuilder); + *gotRow = true; + #ifdef TD_DEBUG_PRINT_ROW STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); @@ -1201,7 +1204,6 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, } // *len = pBuilder->extendedRowSize; - tdSRowEnd(pBuilder); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 7dc8d822e9..3776cb261f 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); #ifdef __cplusplus diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 629a6e8b06..cd5f499c34 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -124,7 +124,7 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { } } -SStreamQueueItem* streamAppendQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { ASSERT(elem); if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; @@ -171,8 +171,8 @@ void streamFreeQitem(SStreamQueueItem* data) { int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* data = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(data); + void* dataStr = taosArrayGetP(pMerge->reqs, i); + taosMemoryFree(dataStr); taosMemoryFree(pRef); } } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d98afa65a..c78ff0756f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -370,80 +370,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat return 0; } -int32_t streamBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet) { - void* buf = NULL; - int32_t code = -1; - int32_t blockNum = taosArrayGetSize(data->blocks); - ASSERT(blockNum != 0); - - SStreamDispatchReq req = { - .streamId = pTask->streamId, - .dataSrcVgId = data->srcVgId, - .upstreamTaskId = pTask->taskId, - .upstreamChildId = pTask->selfChildId, - .upstreamNodeId = pTask->nodeId, - .blockNum = blockNum, - }; - - req.data = taosArrayInit(blockNum, sizeof(void*)); - req.dataLen = taosArrayInit(blockNum, sizeof(int32_t)); - if (req.data == NULL || req.dataLen == NULL) { - goto FAIL; - } - for (int32_t i = 0; i < blockNum; i++) { - SSDataBlock* pDataBlock = taosArrayGet(data->blocks, i); - if (streamAddBlockToDispatchMsg(pDataBlock, &req) < 0) { - goto FAIL; - } - } - int32_t vgId = 0; - int32_t downstreamTaskId = 0; - // find ep - if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - vgId = pTask->fixedEpDispatcher.nodeId; - *ppEpSet = &pTask->fixedEpDispatcher.epSet; - downstreamTaskId = pTask->fixedEpDispatcher.taskId; - } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - // TODO get ctbName for each block - SSDataBlock* pBlock = taosArrayGet(data->blocks, 0); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pBlock->info.groupId); - // TODO: get hash function by hashMethod - - // get groupId, compute hash value - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - // get node - // TODO: optimize search process - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t sz = taosArrayGetSize(vgInfo); - for (int32_t i = 0; i < sz; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { - vgId = pVgInfo->vgId; - downstreamTaskId = pVgInfo->taskId; - *ppEpSet = &pVgInfo->epSet; - break; - } - } - } - - ASSERT(vgId > 0 || vgId == SNODE_HANDLE); - req.taskId = downstreamTaskId; - - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); - - streamDispatchOneReq(pTask, &req, vgId, *ppEpSet); - - code = 0; -FAIL: - if (code < 0 && buf) rpcFreeCont(buf); - if (req.data) taosArrayDestroyP(req.data, (FDelete)taosMemoryFree); - if (req.dataLen) taosArrayDestroy(req.dataLen); - return code; -} - int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); @@ -461,7 +387,7 @@ int32_t streamDispatch(SStreamTask* pTask) { } ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream continue dispatching: task %d", pTask->taskId); + qDebug("stream dispatching: task %d", pTask->taskId); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 72249c5181..06ca26f029 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -82,17 +82,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* return 0; } -#if 0 -static FORCE_INLINE int32_t streamUpdateVer(SStreamTask* pTask, SStreamDataBlock* pBlock) { - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - int32_t childId = pBlock->childId; - int64_t ver = pBlock->sourceVer; - SStreamChildEpInfo* pChildInfo = taosArrayGetP(pTask->childEpInfo, childId); - /*pChildInfo-> = ver;*/ - return 0; -} -#endif - int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) { ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); @@ -150,10 +139,11 @@ int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch) return 0; } + // TODO: handle version int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t cnt = 1; + int32_t batchCnt = 1; void* data = NULL; while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); @@ -169,13 +159,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { } } else { void* newRet; - if ((newRet = streamAppendQueueItem(data, qItem)) == NULL) { + if ((newRet = streamMergeQueueItem(data, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - cnt++; + batchCnt++; data = newRet; - /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ streamQueueProcessSuccess(pTask->inputQueue); } } @@ -198,16 +187,14 @@ int32_t streamExecForAll(SStreamTask* pTask) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, cnt); + qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); streamTaskExecImpl(pTask, data, pRes); qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { - // TODO log failed ver - streamQueueProcessFail(pTask->inputQueue); - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); streamFreeQitem(data); return -1; } @@ -218,17 +205,18 @@ int32_t streamExecForAll(SStreamTask* pTask) { SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; + } else if (((SStreamQueueItem*)data)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; + qRes->childId = pTask->selfChildId; + qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { - // TODO save failed ver - /*streamQueueProcessFail(pTask->inputQueue);*/ taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - taosFreeQitem(qRes); streamFreeQitem(data); + taosFreeQitem(qRes); return -1; } - /*streamQueueProcessSuccess(pTask->inputQueue);*/ } else { taosArrayDestroy(pRes); } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index c686fa05ce..0b1ce27b77 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -124,7 +124,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; pInfo->pCloseWinSBF = NULL; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT); pInfo->pMap = taosHashInit(DEFAULT_MAP_CAPACITY, hashFn, true, HASH_NO_LOCK); pInfo->maxVersion = 0; pInfo->scanGroupId = 0; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 856870caba..3e247e5d79 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -163,6 +163,7 @@ typedef struct SSyncNode { bool changing; int64_t startTime; + int64_t leaderTime; int64_t lastReplicateTime; } SSyncNode; diff --git a/source/libs/sync/inc/syncRespMgr.h b/source/libs/sync/inc/syncRespMgr.h index e37c0bb625..28978af77e 100644 --- a/source/libs/sync/inc/syncRespMgr.h +++ b/source/libs/sync/inc/syncRespMgr.h @@ -32,9 +32,9 @@ typedef struct SRespStub { } SRespStub; typedef struct SSyncRespMgr { - SHashObj * pRespHash; + SHashObj *pRespHash; int64_t ttl; - void * data; + void *data; TdThreadMutex mutex; uint64_t seqNum; } SSyncRespMgr; @@ -46,7 +46,8 @@ int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index); int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub); void syncRespClean(SSyncRespMgr *pObj); -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl); +void syncRespCleanRsp(SSyncRespMgr *pObj); +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp); #ifdef __cplusplus } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 20b4b23bf7..9f9fdf4844 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1100,6 +1100,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { int64_t timeNow = taosGetTimestampMs(); pSyncNode->startTime = timeNow; + pSyncNode->leaderTime = timeNow; pSyncNode->lastReplicateTime = timeNow; syncNodeEventLog(pSyncNode, "sync open"); @@ -2015,6 +2016,8 @@ void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term) { } } +void syncNodeLeaderChangeRsp(SSyncNode* pSyncNode) { syncRespCleanRsp(pSyncNode->pSyncRespMgr); } + void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -2028,6 +2031,14 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // reset elect timer syncNodeResetElectTimer(pSyncNode); + // send rsp to client + syncNodeLeaderChangeRsp(pSyncNode); + + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeFollowerCb != NULL) { + pSyncNode->pFsm->FpBecomeFollowerCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -2063,6 +2074,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { + pSyncNode->leaderTime = taosGetTimestampMs(); + // reset restoreFinish pSyncNode->restoreFinish = false; @@ -2109,6 +2122,11 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLeaderCb != NULL) { + pSyncNode->pFsm->FpBecomeLeaderCb(pSyncNode->pFsm); + } + // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -2944,8 +2962,11 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, } ths->restoreFinish = true; + int64_t restoreDelay = taosGetTimestampMs() - ths->leaderTime; + char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "restore finish, index:%" PRId64, pEntry->index); + snprintf(eventLog, sizeof(eventLog), "restore finish, index:%ld, elapsed:%ld ms, ", pEntry->index, + restoreDelay); syncNodeEventLog(ths, eventLog); } } @@ -3100,7 +3121,7 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries syncNodeEventLog(pSyncNode, logBuf); } - void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { +void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index da04b19e17..d7ed864180 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -108,13 +108,19 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu return 0; // get none object } -void syncRespClean(SSyncRespMgr *pObj) { +void syncRespCleanRsp(SSyncRespMgr *pObj) { taosThreadMutexLock(&(pObj->mutex)); - syncRespCleanByTTL(pObj, pObj->ttl); + syncRespCleanByTTL(pObj, -1, true); taosThreadMutexUnlock(&(pObj->mutex)); } -void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { +void syncRespClean(SSyncRespMgr *pObj) { + taosThreadMutexLock(&(pObj->mutex)); + syncRespCleanByTTL(pObj, pObj->ttl, false); + taosThreadMutexUnlock(&(pObj->mutex)); +} + +void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SRespStub *pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, NULL); int cnt = 0; int sum = 0; @@ -126,12 +132,12 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { while (pStub) { size_t len; - void * key = taosHashGetKey(pStub, &len); + void *key = taosHashGetKey(pStub, &len); uint64_t *pSeqNum = (uint64_t *)key; sum++; int64_t nowMS = taosGetTimestampMs(); - if (nowMS - pStub->createTime > ttl) { + if (nowMS - pStub->createTime > ttl || -1 == ttl) { taosArrayPush(delIndexArray, pSeqNum); cnt++; @@ -148,7 +154,14 @@ void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) { pStub->rpcMsg.pCont = NULL; pStub->rpcMsg.contLen = 0; - pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + + // TODO: and make rpcMsg body, call commit cb + // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &(pStub->rpcMsg), cbMeta); + + pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; + if (pStub->rpcMsg.info.handle != NULL) { + tmsgSendRsp(&(pStub->rpcMsg)); + } } pStub = (SRespStub *)taosHashIterate(pObj->pRespHash, pStub); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ad571a9e82..fad3977a21 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); + wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index c57ea33028..a7d2ba8531 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,8 +16,8 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" -#include "tdef.h" #include "pthread.h" +#include "tdef.h" #ifdef WINDOWS @@ -77,8 +77,8 @@ int32_t tsem_wait(tsem_t* sem) { int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { struct timespec ts, rel; - FILETIME ft_before, ft_after; - int rc; + FILETIME ft_before, ft_after; + int rc; rel.tv_sec = 0; rel.tv_nsec = nanosecs; @@ -218,7 +218,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int e = errno; // if (e == EEXIST) continue; // if (e == EINTR) continue; -// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } while (p->sem == SEM_FAILED); @@ -232,7 +233,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); // if (ret != KERN_SUCCESS) { -// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // // we fail-fast here, because we have less-doc about semaphore_create for the moment // abort(); @@ -259,8 +261,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -271,7 +273,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val -= 1; // if (p->val < 0) { // if (taosThreadCondWait(&p->cond, &p->lock)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -298,8 +301,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // struct tsem_s *p = *sem; // if (!p->valid) { -// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); -// abort(); +// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); abort(); // } // #ifdef SEM_USE_PTHREAD // if (taosThreadMutexLock(&p->lock)) { @@ -310,7 +313,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // p->val += 1; // if (p->val <= 0) { // if (taosThreadCondSignal(&p->cond)) { -// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, +// __func__, // sem); // abort(); // } @@ -333,7 +337,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int tsem_destroy(tsem_t *sem) { // // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); // if (!*sem) { -// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem); +// // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem); // // abort(); // return 0; // } @@ -371,7 +376,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // int r = sem_unlink(name); // if (r) { // int e = errno; -// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, +// fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, +// sem, // e, strerror(e)); // abort(); // } @@ -386,225 +392,189 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // *sem = NULL; // return 0; // } -typedef struct -{ - pthread_mutex_t count_lock; - pthread_cond_t count_bump; - unsigned int count; -}bosal_sem_t; +typedef struct { + pthread_mutex_t count_lock; + pthread_cond_t count_bump; + unsigned int count; +} bosal_sem_t; -int tsem_init(tsem_t *psem, int flags, unsigned int count) -{ - bosal_sem_t *pnewsem; - int result; +int tsem_init(tsem_t *psem, int flags, unsigned int count) { + bosal_sem_t *pnewsem; + int result; - pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); - if (! pnewsem) - { - return -1; - } - result = pthread_mutex_init(&pnewsem->count_lock, NULL); - if (result) - { - free(pnewsem); - return result; - } - result = pthread_cond_init(&pnewsem->count_bump, NULL); - if (result) - { - pthread_mutex_destroy(&pnewsem->count_lock); - free(pnewsem); - return result; - } - pnewsem->count = count; - *psem = (tsem_t)pnewsem; - return 0; + pnewsem = (bosal_sem_t *)malloc(sizeof(bosal_sem_t)); + if (!pnewsem) { + return -1; + } + result = pthread_mutex_init(&pnewsem->count_lock, NULL); + if (result) { + free(pnewsem); + return result; + } + result = pthread_cond_init(&pnewsem->count_bump, NULL); + if (result) { + pthread_mutex_destroy(&pnewsem->count_lock); + free(pnewsem); + return result; + } + pnewsem->count = count; + *psem = (tsem_t)pnewsem; + return 0; } -int tsem_destroy(tsem_t *psem) -{ - bosal_sem_t *poldsem; +int tsem_destroy(tsem_t *psem) { + bosal_sem_t *poldsem; - if (! psem) - { - return EINVAL; - } - poldsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + poldsem = (bosal_sem_t *)*psem; - pthread_mutex_destroy(&poldsem->count_lock); - pthread_cond_destroy(&poldsem->count_bump); - free(poldsem); - return 0; + pthread_mutex_destroy(&poldsem->count_lock); + pthread_cond_destroy(&poldsem->count_bump); + free(poldsem); + return 0; } -int tsem_post(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_post(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - pxsem->count = pxsem->count + 1; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + pxsem->count = pxsem->count + 1; - xresult = pthread_cond_signal(&pxsem->count_bump); + xresult = pthread_cond_signal(&pxsem->count_bump); - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_trywait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_trywait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count > 0) - { - pxsem->count--; - } - else - { - xresult = EAGAIN; - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + if (pxsem->count > 0) { + pxsem->count--; + } else { + xresult = EAGAIN; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_wait(tsem_t *psem) -{ - bosal_sem_t *pxsem; - int result, xresult; +int tsem_wait(tsem_t *psem) { + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + if (pxsem->count == 0) { + xresult = pthread_cond_wait(&pxsem->count_bump, &pxsem->count_lock); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -int tsem_timewait(tsem_t *psem, int64_t nanosecs) -{ +int tsem_timewait(tsem_t *psem, int64_t nanosecs) { struct timespec abstim = { .tv_sec = 0, .tv_nsec = nanosecs, }; - bosal_sem_t *pxsem; - int result, xresult; + bosal_sem_t *pxsem; + int result, xresult; - if (! psem) - { - return EINVAL; - } - pxsem = (bosal_sem_t *)*psem; + if (!psem) { + return EINVAL; + } + pxsem = (bosal_sem_t *)*psem; - result = pthread_mutex_lock(&pxsem->count_lock); - if (result) - { - return result; - } - xresult = 0; + result = pthread_mutex_lock(&pxsem->count_lock); + if (result) { + return result; + } + xresult = 0; - if (pxsem->count == 0) - { - xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + if (pxsem->count == 0) { + xresult = pthread_cond_timedwait(&pxsem->count_bump, &pxsem->count_lock, &abstim); + } + if (!xresult) { + if (pxsem->count > 0) { + pxsem->count--; } - if (! xresult) - { - if (pxsem->count > 0) - { - pxsem->count--; - } - } - result = pthread_mutex_unlock(&pxsem->count_lock); - if (result) - { - return result; - } - if (xresult) - { - errno = xresult; - return -1; - } - return 0; + } + result = pthread_mutex_unlock(&pxsem->count_lock); + if (result) { + return result; + } + if (xresult) { + errno = xresult; + return -1; + } + return 0; } -bool taosCheckPthreadValid(TdThread thread) { +bool taosCheckPthreadValid(TdThread thread) { int32_t ret = taosThreadKill(thread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive return true; - } +} int64_t taosGetSelfPthreadId() { TdThread thread = taosThreadSelf(); @@ -651,7 +621,13 @@ int64_t taosGetSelfPthreadId() { int64_t taosGetPthreadId(TdThread thread) { return (int64_t)thread; } void taosResetPthread(TdThread* thread) { *thread = 0; } bool taosComparePthread(TdThread first, TdThread second) { return first == second; } -int32_t taosGetPId() { return getpid(); } + +int32_t taosGetPId() { + static int32_t pid; + if (pid != 0) return pid; + pid = getpid(); + return pid; +} int32_t taosGetAppName(char* name, int32_t* len) { const char* self = "/proc/self/exe"; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2be1c9f744..8a3d0620ed 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -281,11 +281,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_QUERY, "Topic with invalid query") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Topic with invalid option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_EXIST, "Consumer not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Consumer unchanged") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED, "Topic unchanged") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST, "Subcribe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_OFFSET_NOT_EXIST, "Offset not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") // mnode-stream @@ -623,6 +624,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Invalid index file" //tmq TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") #ifdef TAOS_ERROR_C }; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0bb24826a3..cf7e2d63f0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -237,8 +237,8 @@ ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim -./test.sh -f tsim/stream/session0.sim -./test.sh -f tsim/stream/session1.sim +#./test.sh -f tsim/stream/session0.sim +#./test.sh -f tsim/stream/session1.sim ./test.sh -f tsim/stream/state0.sim ./test.sh -f tsim/stream/triggerInterval0.sim ./test.sh -f tsim/stream/triggerSession0.sim diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py new file mode 100644 index 0000000000..9348a8ca8f --- /dev/null +++ b/tests/system-test/2-query/interp.py @@ -0,0 +1,559 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + dbname = "db" + tbname = "tb" + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute( + f'''create table if not exists {dbname}.{tbname} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + ''' + ) + + tdLog.printNoPrefix("==========step2:insert data") + + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:10', 10, 10, 10, 10, 10.0, 10.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:15', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") + + tdLog.printNoPrefix("==========step3:fill null") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, None) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, None) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(null)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + + tdLog.printNoPrefix("==========step4:fill value") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 1) + tdSql.checkData(11, 0, 15) + tdSql.checkData(12, 0, 1) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(value, 1)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + + tdLog.printNoPrefix("==========step5:fill prev") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 10) + tdSql.checkData(8, 0, 10) + tdSql.checkData(9, 0, 10) + tdSql.checkData(10, 0, 15) + tdSql.checkData(11, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(prev)") + tdSql.checkRows(0) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(prev)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(prev)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(prev)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + + tdLog.printNoPrefix("==========step6:fill next") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkRows(12) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 10) + tdSql.checkData(7, 0, 15) + tdSql.checkData(8, 0, 15) + tdSql.checkData(9, 0, 15) + tdSql.checkData(10, 0, 15) + tdSql.checkData(11, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(next)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 10) + tdSql.checkData(4, 0, 10) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(next)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 10) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 10) + tdSql.checkData(1, 0, 10) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(next)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(next)") + tdSql.checkRows(0) + + + tdLog.printNoPrefix("==========step7:fill linear") + + ## {. . .} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + tdSql.checkData(3, 0, 8) + tdSql.checkData(4, 0, 9) + tdSql.checkData(5, 0, 10) + tdSql.checkData(6, 0, 11) + tdSql.checkData(7, 0, 12) + tdSql.checkData(8, 0, 13) + tdSql.checkData(9, 0, 14) + tdSql.checkData(10, 0, 15) + + ## {} ... + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:01', '2020-02-01 00:00:04') every(1s) fill(linear)") + tdSql.checkRows(0) + + ## {.}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:03', '2020-02-01 00:00:07') every(1s) fill(linear)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 7) + + ## .{}.. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:06', '2020-02-01 00:00:09') every(1s) fill(linear)") + tdSql.checkRows(4) + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 7) + tdSql.checkData(2, 0, 8) + tdSql.checkData(3, 0, 9) + + ## .{.}. + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(1s) fill(linear)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 8) + tdSql.checkData(1, 0, 9) + tdSql.checkData(2, 0, 10) + tdSql.checkData(3, 0, 11) + tdSql.checkData(4, 0, 12) + + ## ..{.} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:13', '2020-02-01 00:00:17') every(1s) fill(linear)") + tdSql.checkRows(3) + tdSql.checkData(0, 0, 13) + tdSql.checkData(1, 0, 14) + tdSql.checkData(2, 0, 15) + + ## ... {} + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") + tdSql.checkRows(0) + + + tdLog.printNoPrefix("==========step8:test intra block interpolation") + tdSql.execute(f"drop database {dbname}"); + + tdSql.prepare() + + tdSql.execute(f"create table if not exists {dbname}.{tbname} (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10))") + + # set two data point has 10 days interval will be stored in different datablocks + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-01 00:00:05', 5, 5, 5, 5, 5.0, 5.0, true, 'varchar', 'nchar')") + tdSql.execute(f"insert into {dbname}.{tbname} values ('2020-02-11 00:00:05', 15, 15, 15, 15, 15.0, 15.0, true, 'varchar', 'nchar')") + + tdSql.execute(f"flush database {dbname}"); + + # test fill null + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(null)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + tdSql.checkData(6, 0, None) + tdSql.checkData(7, 0, None) + tdSql.checkData(8, 0, None) + tdSql.checkData(9, 0, None) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(null)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(null)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, None) + tdSql.checkData(3, 0, None) + tdSql.checkData(4, 0, None) + tdSql.checkData(5, 0, None) + + + # test fill value + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 1) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 1) + tdSql.checkData(9, 0, 1) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 1) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 1) + + + # test fill prev + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 5) + tdSql.checkData(7, 0, 5) + tdSql.checkData(8, 0, 5) + tdSql.checkData(9, 0, 5) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 5) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 5) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(prev)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + + # test fill next + + ## | {. | | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-11 00:00:05') every(1d) fill(next)") + tdSql.checkRows(11) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + tdSql.checkData(6, 0, 15) + tdSql.checkData(7, 0, 15) + tdSql.checkData(8, 0, 15) + tdSql.checkData(9, 0, 15) + tdSql.checkData(10, 0, 15) + + ## | . | {} | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-03 00:00:05', '2020-02-07 00:00:05') every(1d) fill(next)") + tdSql.checkRows(5) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + + ## | {. | } | . | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-01-31 00:00:05', '2020-02-05 00:00:05') every(1d) fill(next)") + tdSql.checkRows(6) + tdSql.checkData(0, 0, 5) + tdSql.checkData(1, 0, 5) + tdSql.checkData(2, 0, 15) + tdSql.checkData(3, 0, 15) + tdSql.checkData(4, 0, 15) + tdSql.checkData(5, 0, 15) + + ## | . | { | .} | + tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(next)") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 15) + tdSql.checkData(1, 0, 15) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index f09265c300..cb74c33d92 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -335,7 +335,7 @@ class TDTestCase: # case11 = {"alias": ", st1"} # self.checksample(**case11) tdSql.query("select sample( c1 , 1 ) , st1 from t1") - + # case12 = {"alias": ", c1"} # self.checksample(**case12) @@ -497,7 +497,7 @@ class TDTestCase: # tdSql.query(" select sample(c1 , 1) + 2 from t1 ") err41 = {"alias": "+ avg(c1)"} # self.checksample(**err41) # mix with arithmetic 2 - + # err42 = {"alias": ", c1"} # self.checksample(**err42) tdSql.query("select sample( c1 , 1 ) , c1 from t1") @@ -605,14 +605,14 @@ class TDTestCase: tdSql.execute(f"create table tt{i} using stb2 tags({i})") pass - + def check_sample(self , sample_query , origin_query ): tdSql.query(origin_query) origin_datas = tdSql.queryResult - + tdSql.query(sample_query) sample_datas = tdSql.queryResult @@ -620,7 +620,7 @@ class TDTestCase: for ind , sample_data in enumerate(sample_datas): if sample_data not in origin_datas: status = False - + if status: tdLog.info(" sample data is in datas groups ,successed sql is : %s" % sample_query ) else: @@ -637,7 +637,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -689,7 +689,7 @@ class TDTestCase: tdSql.error(" select sample(c1,ts) from t1 ") tdSql.error(" select sample(c1,false) from t1 ") tdSql.query(" select sample(123,1) from t1 ") - + tdSql.query(" select sample(c1,2) from t1 ") tdSql.checkRows(2) tdSql.query(" select sample(c1,10) from t1 ") @@ -704,10 +704,10 @@ class TDTestCase: tdSql.checkRows(9) tdSql.error(" select sample(c1,-1) from t1 ") - # bug need fix + # bug need fix # tdSql.query("select sample(c1 ,2) , 123 from stb1;") - # all type support + # all type support tdSql.query(" select sample(c1 , 20 ) from ct4 ") tdSql.checkRows(9) @@ -761,7 +761,7 @@ class TDTestCase: self.check_sample("select sample( c1 ,3 ) from t1 where c1 between 1 and 10" ,"select c1 from t1 where c1 between 1 and 10") - # join + # join tdSql.query("select sample( ct4.c1 , 1 ) from ct1, ct4 where ct4.ts=ct1.ts") @@ -772,22 +772,22 @@ class TDTestCase: self.check_sample("select sample(c1,2) from stb1 partition by tbname" , "select c1 from stb1 partition by tbname") - # nest query + # nest query # tdSql.query("select sample(c1,2) from (select c1 from t1); ") # tdSql.checkRows(2) - # union all + # union all tdSql.query("select sample(c1,2) from t1 union all select sample(c1,3) from t1") tdSql.checkRows(5) # fill interval - # not support mix with other function + # not support mix with other function tdSql.error("select top(c1,2) , sample(c1,2) from ct1") tdSql.error("select max(c1) , sample(c1,2) from ct1") tdSql.query("select c1 , sample(c1,2) from ct1") - # bug for mix with scalar + # bug for mix with scalar tdSql.query("select 123 , sample(c1,100) from ct1") tdSql.query("select sample(c1,100)+2 from ct1") tdSql.query("select abs(sample(c1,100)) from ct1") @@ -864,13 +864,13 @@ class TDTestCase: for i in range(2000): ts = self.ts+i*10 tdSql.execute(f"insert into sub_tb values({ts} ,{i})") - + tdSql.query("select count(*) from st") tdSql.checkData(0,0,2000) tdSql.query("select sample(c1 ,1000) from st") tdSql.checkRows(1000) - # bug need fix + # bug need fix tdSql.query("select c1 ,t1, sample(c1,2) from db.stb1 partition by c1 ") tdSql.query("select sample(c1,2) from db.stb1 partition by c1 ") # tdSql.query("select c1 ,ind, sample(c1,2) from sample_db.st partition by c1 ") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 92aa21127b..596dabfa88 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -109,6 +109,9 @@ python3 ./test.py -f 2-query/function_stateduration.py -R python3 ./test.py -f 2-query/histogram.py python3 ./test.py -f 2-query/histogram.py -R +python3 ./test.py -f 2-query/interp.py +python3 ./test.py -f 2-query/interp.py -R + python3 ./test.py -f 1-insert/update_data.py @@ -343,6 +346,7 @@ python3 ./test.py -f 2-query/arcsin.py -Q 2 python3 ./test.py -f 2-query/arccos.py -Q 2 python3 ./test.py -f 2-query/arctan.py -Q 2 python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +python3 ./test.py -f 2-query/interp.py -Q 2 # python3 ./test.py -f 2-query/nestedQuery.py -Q 2 # python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 @@ -462,3 +466,4 @@ python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 python3 ./test.py -f 2-query/sml.py -Q 3 +python3 ./test.py -f 2-query/interp.py -Q 3 diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1f066daabe..5751c347e3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -112,7 +112,7 @@ ELSE () COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X github.com/taosdata/taosadapter/version.Version=${taos_version} -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND wget -c https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : + COMMAND wget -nc https://github.com/upx/upx/releases/download/v3.96/upx-3.96-${PLATFORM_ARCH_STR}_linux.tar.xz -O $ENV{HOME}/upx.tar.xz && tar -xvJf $ENV{HOME}/upx.tar.xz -C $ENV{HOME}/ --strip-components 1 > /dev/null && $ENV{HOME}/upx taosadapter || : COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 88ef46c5d6..818cc5425b 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -38,7 +38,7 @@ #define SHELL_STARTUP "Check the details of the service status." #define SHELL_WIDTH "Set the default binary display width, default is 30." #define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." -#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes." +#define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." #define SHELL_EMAIL "" @@ -62,7 +62,7 @@ void shellPrintHelp() { printf("%s%s%s%s\r\n", indent, "-f,", indent, SHELL_FILE); printf("%s%s%s%s\r\n", indent, "-h,", indent, SHELL_HOST); printf("%s%s%s%s\r\n", indent, "-k,", indent, SHELL_CHECK); - printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKG_LEN); + printf("%s%s%s%s\r\n", indent, "-l,", indent, SHELL_PKT_LEN); printf("%s%s%s%s\r\n", indent, "-n,", indent, SHELL_NET_ROLE); printf("%s%s%s%s\r\n", indent, "-N,", indent, SHELL_PKT_NUM); printf("%s%s%s%s\r\n", indent, "-p,", indent, SHELL_PASSWORD); @@ -105,11 +105,11 @@ static struct argp_option shellOptions[] = { {"startup", 't', 0, 0, SHELL_STARTUP}, {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, - {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, + {"pktlen", 'l', "PKTLEN", 0, SHELL_PKT_LEN}, #ifdef WEBSOCKET {"dsn", 'E', "DSN", 0, SHELL_DSN}, {"restful", 'R', 0, 0, SHELL_REST}, - {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, + {"timeout", 'T', "SECONDS", 0, SHELL_TIMEOUT}, #endif {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, {0}, @@ -228,7 +228,7 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } diff --git a/tools/taos-tools b/tools/taos-tools deleted file mode 160000 index 3c7dafeea3..0000000000 --- a/tools/taos-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3c7dafeea3e558968165b73bee0f51024898e3da