diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 249a8d1c9d..6fa3483099 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -78,7 +78,7 @@ def check_docs(){ file_only_tdgpt_change_except = sh ( script: ''' cd ${WKC} - git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v "forecastoperator.c\\|anomalywindowoperator.c" |grep -v "tsim/analytics" |grep -v "tdgpt_cases.task" || : + git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${CHANGE_TARGET}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -v "forecastoperator.c\\|anomalywindowoperator.c\\|tanalytics.h\\|tanalytics.c" |grep -v "tsim/analytics" |grep -v "tdgpt_cases.task" || : ''', returnStdout: true ).trim() diff --git a/docs/en/10-third-party/03-visual/01-grafana.md b/docs/en/10-third-party/03-visual/01-grafana.md index ec857f7795..8a503b4195 100644 --- a/docs/en/10-third-party/03-visual/01-grafana.md +++ b/docs/en/10-third-party/03-visual/01-grafana.md @@ -22,15 +22,11 @@ import imgStep11 from '../../assets/grafana-11.png'; This document describes how to integrate the TDengine data source with the open-source data visualization system [Grafana](https://www.grafana.com/) to achieve data visualization and build a monitoring and alert system. With the TDengine plugin, you can easily display data from TDengine tables on Grafana dashboards without the need for complex development work. -## Grafana Version Requirements - -TDengine currently supports Grafana version 7.5 and above. It is recommended to use the latest version. Please download and install the corresponding version of Grafana according to your system environment. - ## Prerequisites To add the TDengine data source to Grafana normally, the following preparations are needed. -- Grafana service has been deployed and is running normally. +- Grafana service has been deployed and is running normally. TDengine currently supports Grafana version 7.5 and above. It is recommended to use the latest version. **Note**: Ensure that the account starting Grafana has write permissions to its installation directory, otherwise you may not be able to install plugins later. - TDengine cluster has been deployed and is running normally. - taosAdapter has been installed and is running normally. For details, please refer to the [taosAdapter user manual](../../../tdengine-reference/components/taosadapter/) diff --git a/docs/en/10-third-party/05-bi/09-seeq.md b/docs/en/10-third-party/05-bi/09-seeq.md index c8f7462a19..7fb7569461 100644 --- a/docs/en/10-third-party/05-bi/09-seeq.md +++ b/docs/en/10-third-party/05-bi/09-seeq.md @@ -13,13 +13,11 @@ Seeq is advanced analytics software for the manufacturing and Industrial Interne Through the TDengine Java connector, Seeq can easily support querying time-series data provided by TDengine and offer data presentation, analysis, prediction, and other functions. -## Seeq Installation Method +## Prerequisites -Download the relevant software from [Seeq's official website](https://www.seeq.com/customer-download), such as Seeq Server and Seeq Data Lab, etc. Seeq Data Lab needs to be installed on a different server from Seeq Server and interconnected through configuration. For detailed installation and configuration instructions, refer to the [Seeq Knowledge Base](https://support.seeq.com/kb/latest/cloud/). +- Seeq has been installed. Download the relevant software from [Seeq's official website](https://www.seeq.com/customer-download), such as Seeq Server and Seeq Data Lab, etc. Seeq Data Lab needs to be installed on a different server from Seeq Server and interconnected through configuration. For detailed installation and configuration instructions, refer to the [Seeq Knowledge Base](https://support.seeq.com/kb/latest/cloud/). -### TDengine Local Instance Installation Method - -Please refer to the [official documentation](../../../get-started). +- TDengine local instance has been installed. Please refer to the [official documentation](../../../get-started). If using TDengine Cloud, please go to https://cloud.taosdata.com apply for an account and log in to see how to access TDengine Cloud. ## Configuring Seeq to Access TDengine diff --git a/docs/en/14-reference/09-error-code.md b/docs/en/14-reference/09-error-code.md index 1d3ea3f9a1..2bbd8f9305 100644 --- a/docs/en/14-reference/09-error-code.md +++ b/docs/en/14-reference/09-error-code.md @@ -386,7 +386,7 @@ This document details the server error codes that may be encountered when using | 0x8000260D | Tags number not matched | Mismatched number of tag columns | Check and correct the SQL statement | | 0x8000260E | Invalid tag name | Invalid or non-existent tag name | Check and correct the SQL statement | | 0x80002610 | Value is too long | Value length exceeds limit | Check and correct the SQL statement or API parameters | -| 0x80002611 | Password can not be empty | Password is empty | Use a valid password | +| 0x80002611 | Password too short or empty | Password is empty or less than 8 chars | Use a valid password | | 0x80002612 | Port should be an integer that is less than 65535 and greater than 0 | Illegal port number | Check and correct the port number | | 0x80002613 | Endpoint should be in the format of 'fqdn:port' | Incorrect address format | Check and correct the address information | | 0x80002614 | This statement is no longer supported | Feature has been deprecated | Refer to the feature documentation | diff --git a/docs/zh/08-operation/09-backup.md b/docs/zh/08-operation/09-backup.md index 1eda0a646b..075cc244f4 100644 --- a/docs/zh/08-operation/09-backup.md +++ b/docs/zh/08-operation/09-backup.md @@ -6,45 +6,154 @@ toc_max_heading_level: 4 为了防止数据丢失、误删操作，TDengine 提供全面的数据备份、恢复、容错、异地数据实时同步等功能，以保证数据存储的安全。本节简要说明备份和恢复功能。 -## 基于 taosdump 进行数据备份恢复 +# 1. 基于 taosdump 进行数据备份恢复 -taosdump 是一个开源工具，用于支持从运行中的 TDengine 集群备份数据并将备份的数据恢复到相同或另一个正在运行的 TDengine 集群中。taosdump 可以将数据库作为逻辑数据单元进行备份，也可以对数据库中指定时间段内的数据记录进行备份。在使用taosdump 时，可以指定数据备份的目录路径。如果不指定目录路径，taosdump 将默认将数据备份到当前目录。 +taosdump 是一个开源工具，用于支持从运行中的 TDengine 集群备份数据并将备份的数据恢复到相同或另一个正在运行的 TDengine +集群中。taosdump 可以将数据库作为逻辑数据单元进行备份，也可以对数据库中指定时间段内的数据记录进行备份。在使用taosdump +时，可以指定数据备份的目录路径。如果不指定目录路径，taosdump 将默认将数据备份到当前目录。 以下为 taosdump 执行数据备份的使用示例。 + ```shell taosdump -h localhost -P 6030 -D dbname -o /file/path ``` -执行上述命令后，taosdump 会连接 localhost:6030 所在的 TDengine 集群，查询数据库 dbname 中的所有数据，并将数据备份到 /f ile/path 下。 +执行上述命令后，taosdump 会连接 localhost:6030 所在的 TDengine 集群，查询数据库 dbname 中的所有数据，并将数据备份到 /f +ile/path 下。 -在使用 taosdump 时，如果指定的存储路径已经包含数据文件，taosdump 会提示用户并立即退出，以避免数据被覆盖。这意味着同一存储路径只能用于一次备份。如果你看到相关提示，请谨慎操作，以免误操作导致数据丢失。 +在使用 taosdump 时，如果指定的存储路径已经包含数据文件，taosdump +会提示用户并立即退出，以避免数据被覆盖。这意味着同一存储路径只能用于一次备份。如果你看到相关提示，请谨慎操作，以免误操作导致数据丢失。 + +要将本地指定文件路径中的数据文件恢复到正在运行的 TDengine 集群中，可以通过指定命令行参数和数据文件所在路径来执行 taosdump +命令。以下为 taosdump 执行数据恢复的示例代码。 -要将本地指定文件路径中的数据文件恢复到正在运行的 TDengine 集群中，可以通过指定命令行参数和数据文件所在路径来执行 taosdump 命令。以下为 taosdump 执行数据恢复的示例代码。 ```shell taosdump -i /file/path -h localhost -P 6030 ``` 执行上述命令后，taosdump 会连接 localhost:6030 所在的 TDengine 集群，并将 /file/path 下的数据文件恢复到 TDengine 集群中。 -## 基于 TDengine Enterprise 进行数据备份恢复 +# 2. 基于 TDengine Enterprise 进行数据备份恢复 -TDengine Enterprise 提供了一个高效的增量备份功能，具体流程如下。 +## 2.1. 概念 -第 1 步，通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 +基于 TDengine 的数据订阅功能，TDengine Enterprise 实现了数据的增量备份和恢复。用户可以通过 taosExplorer 对 TDengine +集群进行备份和恢复。 -第 2 步，在 taosExplorer 服务页面中的“系统管理 - 备份”页面新增一个数据备份任务，在任务配置信息中填写需要备份的数据库名称和备份存储文件路径，完成创建任务 -后即可启动数据备份。 在数据备份配置页面中可以配置三个参数： - - 备份周期：必填项，配置每次执行数据备份的时间间隔，可通过下拉框选择每天、每 7 天、每 30 天执行一次数据备份，配置后，会在对应的备份周期的0:00时启动一次数据备份任务； - - 数据库：必填项，配置需要备份的数据库名（数据库的 wal_retention_period 参数需大于0）； - - 目录：必填项，配置将数据备份到 taosX 所在运行环境中指定的路径下，如 /root/data_backup； +TDengine Enterprise 的备份和恢复功能包括以下几个概念： -第 3 步，在数据备份任务完成后，在相同页面的已创建任务列表中找到创建的数据备份任务，直接执行一键恢复，就能够将数据恢复到 TDengine 中。 +1. 备份对象：用户可以对一个数据库，或者一个超级表进行备份。 +2. 备份计划：用户可以为某个备份对象创建一个备份计划。备份计划从指定的时间点开始，周期性的执行一次备份任务，并生成一组备份文件。 +3. 备份点：每执行一次备份任务，生成一组备份文件，它们对应一个时间点，称为**备份点**。第一个备份点称为**初始备份点**。 +4. 备份文件：多个备份点，组成备份计划的备份文件。 +5. 恢复任务：用户可以选择备份计划的某个备份点，创建一个恢复任务。恢复任务会从初始备份点开始，逐个应用备份点，恢复到指定的备份点。 -与 taosdump 相比，如果对相同的数据在指定存储路径下进行多次备份操作，由于TDengine Enterprise 不仅备份效率高，而且实行的是增量处理，因此每次备份任务都会很快完成。而由于 taosdump 永远是全量备份，因此 TDengine Enterprise 在数据量较大的场景下可以显著减小系统开销，而且更加方便。 +![backup-zh-00.png](./pic/backup-00-concept.png "数据备份和恢复") -**常见错误排查** +以上面的图为例： -1. 如果任务启动失败并报以下错误： +1. 用户创建了一个备份计划，从 2024-08-27 00:00:00 开始，每隔 1 天执行一次备份任务。 +2. 在 2024-08-27 00:00:00 执行了第一次备份任务，生成了一个备份点。 +3. 之后，每隔 1 天执行一次备份任务，生成了多个备份点，组成了备份文件。 +4. 用户可以选择某个备份点，创建一个恢复任务，恢复到指定的备份点。 +5. 恢复任务会从初始备份点开始，逐个应用备份点，恢复到指定的备份点。 + +## 2.2. 备份计划 + +### 2.1.1. 创建 + +1. 通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 +2. 在 taosExplorer 服务页面中，进入“系统管理 - 备份”页面，点击“创建备份计划”按钮。 + +![backup-zh-01.png](./pic/backup-01-create.png "创建备份计划") + +3. 在弹出的“创建备份计划”表单中，填写备份计划的相关信息。 + +![backup-zh-02.png](./pic/backup-02-form.png "填写备份计划信息") + +需要填写的信息包括： + +* 数据库：需要备份的数据库名称。一个备份计划只能备份一个数据库/超级表。 +* 超级表：需要备份的超级表名称。如果不填写，则备份整个数据库。 +* 下次执行时间：首次执行备份任务的日期时间。 +* 备份周期：备份点之间的时间间隔。注意：备份周期必须大于数据库的 WAL_RETENTION_PERIOD 参数值。 +* 错误重试次数：对于可通过重试解决的错误，系统会按照此次数进行重试。 +* 错误重试间隔：每次重试之间的时间间隔。 +* 目录：存储备份文件的目录。 +* 备份文件大小：备份文件的大小限制。当备份文件大小达到此限制时，会自动创建新的备份文件。 +* 文件压缩等级：备份文件的压缩等级。支持：最快速度、最佳压缩比、兼具速度和压缩比。 + +创建成功后，备份计划会开始按照配置的参数运行。 + +### 2.1.2. 查看 + +在“备份计划”下的列表中，可以查看已创建的备份计划。 + +![backup-zh-03.png](./pic/backup-03-list.png "查看备份计划列表") + +点击“操作”中的“查看”按钮，可以查看备份计划的详细信息。 + +![backup-zh-04.png](./pic/backup-04-view.png "查看备份计划详情") + +### 2.1.3. 修改 + +点击“操作”中的“修改”按钮，可以修改备份计划的配置。 + +![backup-zh-05.png](./pic/backup-05-edit.png "修改备份计划") + +修改备份计划的配置后，当前运行的备份任务会先停止，然后按照新的配置重新运行。 + +### 2.1.4. 复制 + +点击“操作”中的“复制”按钮，可以复制备份计划。 + +![backup-zh-06.png](./pic/backup-06-copy.png "复制备份计划") + +除了数据库和超级表被置为空外，其他配置项和被复制的计划相同。用户点击“确认”后，创建一个新的备份计划。 + +### 2.1.5. 删除 + +在操作中点击关闭按钮，可以停止当前备份计划。点击“操作”中的“删除”按钮，可以删除备份计划。 + +![backup-zh-07.png](./pic/backup-07-del.png "删除备份计划") + +删除备份计划时，可以选择，是否删除关联的备份文件。 + +## 2.2. 备份文件 + +### 2.2.1. 查看 + +在备份计划列表中，选择要一个备份计划。在“备份文件”列中，点击“查看”按钮。可以查看和备份计划的所有备份点。 + +![backup-zh-08.png](./pic/backup-08-files.png "查看备份文件") + +在备份文件列表中，可以查看备份文件的详细信息。 + +![backup-zh-09.png](./pic/backup-09-filelist.png "查看备份文件列表") + +## 2.3. 恢复任务 + +### 2.3.1. 创建 + +在备份文件列表中，点击“操作”中的“恢复”按钮，可以创建一个恢复任务。 + +![backup-zh-10.png](./pic/backup-10-restore-create.png "创建恢复任务") + +在弹出的对话框中，选择使用哪个备份点开始恢复，默认为最早的备份点。点击“确定”后，创建恢复任务，并跳转至“恢复任务”列表。 + +### 2.3.2. 查看 + +在“恢复任务”列表中，可以查看已创建的恢复任务。 + +![backup-zh-11.png](./pic/backup-11-restore-list.png "查看恢复任务列表") + +恢复任务可以终止。点击“操作”中的开关，可以终止当前恢复任务。 + +# 3. 常见错误排查 + +## 3.1. 端口访问异常 + +如果任务启动失败并报以下错误： ```text Error: tmq to td task exec error @@ -52,9 +161,12 @@ Error: tmq to td task exec error Caused by: [0x000B] Unable to establish connection ``` + 产生原因是与数据源的端口链接异常，需检查数据源 FQDN 是否联通及端口 6030 是否可正常访问。 -2. 如果使用 WebSocket 连接，任务启动失败并报以下错误： +## 3.2. 连接异常 + +如果使用 WebSocket 连接，任务启动失败并报以下错误： ```text Error: tmq to td task exec error @@ -73,7 +185,9 @@ Caused by: - "HTTP error: *": 可能连接到错误的 taosAdapter 端口或 LSB/Nginx/Proxy 配置错误。 - "WebSocket protocol error: Handshake not finished": WebSocket 连接错误，通常是因为配置的端口不正确。 -3. 如果任务启动失败并报以下错误： +## 3.3. 任务启动失败 + +如果任务启动失败并报以下错误： ```text Error: tmq to td task exec error @@ -88,5 +202,6 @@ Caused by: 修改数据 WAL 配置： ```sql -alter database test wal_retention_period 3600; +alter +database test wal_retention_period 3600; ``` \ No newline at end of file diff --git a/docs/zh/08-operation/pic/backup-00-concept.png b/docs/zh/08-operation/pic/backup-00-concept.png new file mode 100644 index 0000000000..5123b4d540 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-00-concept.png differ diff --git a/docs/zh/08-operation/pic/backup-01-create.png b/docs/zh/08-operation/pic/backup-01-create.png new file mode 100644 index 0000000000..a424c276e5 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-01-create.png differ diff --git a/docs/zh/08-operation/pic/backup-02-form.png b/docs/zh/08-operation/pic/backup-02-form.png new file mode 100644 index 0000000000..3ccd81c831 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-02-form.png differ diff --git a/docs/zh/08-operation/pic/backup-03-list.png b/docs/zh/08-operation/pic/backup-03-list.png new file mode 100644 index 0000000000..505d6a8040 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-03-list.png differ diff --git a/docs/zh/08-operation/pic/backup-04-view.png b/docs/zh/08-operation/pic/backup-04-view.png new file mode 100644 index 0000000000..7bfa699906 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-04-view.png differ diff --git a/docs/zh/08-operation/pic/backup-05-edit.png b/docs/zh/08-operation/pic/backup-05-edit.png new file mode 100644 index 0000000000..5ff1204ad5 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-05-edit.png differ diff --git a/docs/zh/08-operation/pic/backup-06-copy.png b/docs/zh/08-operation/pic/backup-06-copy.png new file mode 100644 index 0000000000..2ec1ea68d0 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-06-copy.png differ diff --git a/docs/zh/08-operation/pic/backup-07-del.png b/docs/zh/08-operation/pic/backup-07-del.png new file mode 100644 index 0000000000..e1cf4748bf Binary files /dev/null and b/docs/zh/08-operation/pic/backup-07-del.png differ diff --git a/docs/zh/08-operation/pic/backup-08-files.png b/docs/zh/08-operation/pic/backup-08-files.png new file mode 100644 index 0000000000..07f2184d4f Binary files /dev/null and b/docs/zh/08-operation/pic/backup-08-files.png differ diff --git a/docs/zh/08-operation/pic/backup-09-filelist.png b/docs/zh/08-operation/pic/backup-09-filelist.png new file mode 100644 index 0000000000..b963091f36 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-09-filelist.png differ diff --git a/docs/zh/08-operation/pic/backup-10-restore-create.png b/docs/zh/08-operation/pic/backup-10-restore-create.png new file mode 100644 index 0000000000..e0e22160d0 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-10-restore-create.png differ diff --git a/docs/zh/08-operation/pic/backup-11-restore-list.png b/docs/zh/08-operation/pic/backup-11-restore-list.png new file mode 100644 index 0000000000..ca1f1b45d5 Binary files /dev/null and b/docs/zh/08-operation/pic/backup-11-restore-list.png differ diff --git a/docs/zh/10-third-party/03-visual/01-grafana.mdx b/docs/zh/10-third-party/03-visual/01-grafana.mdx index d7406352c9..043cfcaa5c 100644 --- a/docs/zh/10-third-party/03-visual/01-grafana.mdx +++ b/docs/zh/10-third-party/03-visual/01-grafana.mdx @@ -10,15 +10,11 @@ import TabItem from "@theme/TabItem"; ## 概述 本文档介绍如何将 TDengine 数据源与开源数据可视化系统 [Grafana](https://www.grafana.com/) 集成，以实现数据的可视化和监测报警系统的搭建。通过 TDengine 插件，您可以轻松地将 TDengine 数据表的数据展示在 Grafana 仪表盘上，且无需进行复杂的开发工作。 -## Grafana 版本要求 -当前 TDengine 支持 Grafana 7.5 及以上版本，建议使用最新版本。请根据您的系统环境下载并安装对应版本的 Grafana。 - - ## 前置条件 要让 Grafana 能正常添加 TDengine 数据源，需要以下几方面的准备工作。 -- Grafana 服务已经部署并正常运行。 +- Grafana 服务已经部署并正常运行。当前 TDengine 支持 Grafana 7.5 及以上版本，建议使用最新版本。 **注意**：要确保启动 Grafana 的账号有其安装目录的写权限，否则可能后面无法安装插件。 - TDengine 集群已经部署并正常运行。 - taosAdapter 已经安装并正常运行。具体细节请参考 [taosAdapter 的使用手册](../../../reference/components/taosadapter) diff --git a/docs/zh/10-third-party/05-bi/05-yhbi.md b/docs/zh/10-third-party/05-bi/05-yhbi.md index b60b0495f0..70dda71051 100644 --- a/docs/zh/10-third-party/05-bi/05-yhbi.md +++ b/docs/zh/10-third-party/05-bi/05-yhbi.md @@ -10,13 +10,10 @@ toc_max_heading_level: 4 一旦数据源配置完成，永洪BI便能直接从TDengine中读取数据，并利用其强大的数据处理和分析功能，为用户提供丰富的数据展示、分析和预测能力。这意味着用户无须编写复杂的代码或进行烦琐的数据转换工作，即可轻松获取所需的业务洞察。 -## 安装永洪 BI +## 前置条件 -确保永洪 BI 已经安装并运行（如果未安装，请到永洪科技官方下载页面下载）。 - -## 安装JDBC驱动 - -从 maven.org 下载 TDengine JDBC 连接器文件 “taos-jdbcdriver-3.2.7-dist.jar”，并安装在永洪 BI 的机器上。 +- 确保永洪 BI 已经安装并运行（如果未安装，请到永洪科技官方下载页面下载）。 +- 安装JDBC驱动。从 maven.org 下载 TDengine JDBC 连接器文件 “taos-jdbcdriver-3.4.0-dist.jar”，并安装在永洪 BI 的机器上。 ## 配置JDBC数据源 diff --git a/docs/zh/10-third-party/05-bi/09-seeq.md b/docs/zh/10-third-party/05-bi/09-seeq.md index 7e61cdcb11..e01deb7e84 100644 --- a/docs/zh/10-third-party/05-bi/09-seeq.md +++ b/docs/zh/10-third-party/05-bi/09-seeq.md @@ -8,16 +8,11 @@ Seeq 是制造业和工业互联网（IIOT）高级分析软件。Seeq 支持在 通过 TDengine Java connector， Seeq 可以轻松支持查询 TDengine 提供的时序数据，并提供数据展现、分析、预测等功能。 -## Seeq 安装方法 +## 前置条件 -从 [Seeq 官网](https://www.seeq.com/customer-download)下载相关软件，例如 Seeq Server 和 Seeq Data Lab 等。Seeq Data Lab 需要安装在和 Seeq Server 不同的服务器上，并通过配置和 Seeq Server 互联。详细安装配置指令参见[Seeq 知识库]( https://support.seeq.com/kb/latest/cloud/)。 +- Seeq 已经安装。从 [Seeq 官网](https://www.seeq.com/customer-download)下载相关软件，例如 Seeq Server 和 Seeq Data Lab 等。Seeq Data Lab 需要安装在和 Seeq Server 不同的服务器上，并通过配置和 Seeq Server 互联。详细安装配置指令参见[Seeq 知识库]( https://support.seeq.com/kb/latest/cloud/)。 -### TDengine 本地实例安装方法 - -请参考[官网文档](../../../get-started)。 - -### TDengine Cloud 访问方法 -如果使用 Seeq 连接 TDengine Cloud，请在 https://cloud.taosdata.com 申请帐号并登录查看如何访问 TDengine Cloud。 +- TDengine 本地实例已安装。 请参考[官网文档](../../../get-started)。 若使用 TDengine Cloud，请在 https://cloud.taosdata.com 申请帐号并登录查看如何访问 TDengine Cloud。 ## 配置 Seeq 访问 TDengine diff --git a/docs/zh/14-reference/09-error-code.md b/docs/zh/14-reference/09-error-code.md index 2bebe2406b..51453cef4c 100644 --- a/docs/zh/14-reference/09-error-code.md +++ b/docs/zh/14-reference/09-error-code.md @@ -403,7 +403,7 @@ description: TDengine 服务端的错误码列表和详细说明 | 0x8000260D | Tags number not matched | tag列个数不匹配 | 检查并修正SQL语句 | | 0x8000260E | Invalid tag name | 无效或不存在的tag名 | 检查并修正SQL语句 | | 0x80002610 | Value is too long | 值长度超出限制 | 检查并修正SQL语句或API参数 | -| 0x80002611 | Password can not be empty | 密码为空 | 使用合法的密码 | +| 0x80002611 | Password too short or empty | 密码为空或少于 8 个字符 | 使用合法的密码 | | 0x80002612 | Port should be an integer that is less than 65535 and greater than 0 | 端口号非法 | 检查并修正端口号 | | 0x80002613 | Endpoint should be in the format of 'fqdn:port' | 地址格式错误 | 检查并修正地址信息 | | 0x80002614 | This statement is no longer supported | 功能已经废弃 | 参考功能文档说明 | diff --git a/include/common/tanalytics.h b/include/common/tanalytics.h index d0af84ecfb..6ebdb38fa6 100644 --- a/include/common/tanalytics.h +++ b/include/common/tanalytics.h @@ -28,8 +28,8 @@ extern "C" { #define ANAL_FORECAST_DEFAULT_ROWS 10 #define ANAL_FORECAST_DEFAULT_CONF 95 #define ANAL_FORECAST_DEFAULT_WNCHECK 1 -#define ANAL_FORECAST_MAX_ROWS 10000 -#define ANAL_ANOMALY_WINDOW_MAX_ROWS 10000 +#define ANAL_FORECAST_MAX_ROWS 40000 +#define ANAL_ANOMALY_WINDOW_MAX_ROWS 40000 typedef struct { EAnalAlgoType type; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e6333d2ddc..3ea931c42c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -114,6 +114,7 @@ extern int32_t tsRetentionSpeedLimitMB; extern const char *tsAlterCompactTaskKeywords; extern int32_t tsNumOfCompactThreads; +extern int32_t tsNumOfRetentionThreads; // sync raft extern int32_t tsElectInterval; @@ -291,6 +292,7 @@ extern bool tsFilterScalarMode; extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; extern int32_t tsResolveFQDNRetryTime; +extern bool tsStreamCoverage; extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 52cb847b6f..2c7e6216f5 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -109,8 +109,9 @@ int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInter int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); // input: interbuf1, interbuf2 // output: resultBuf -int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, - SUdfInterBuf *resultBuf); +// udf todo: aggmerge +// int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, +// SUdfInterBuf *resultBuf); // input: block // output: resultData int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 64ef0b3829..e317fdd65a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -801,7 +801,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_PAR_TAGS_NOT_MATCHED TAOS_DEF_ERROR_CODE(0, 0x260D) #define TSDB_CODE_PAR_INVALID_TAG_NAME TAOS_DEF_ERROR_CODE(0, 0x260E) #define TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x2610) -#define TSDB_CODE_PAR_PASSWD_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) +#define TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY TAOS_DEF_ERROR_CODE(0, 0x2611) #define TSDB_CODE_PAR_INVALID_PORT TAOS_DEF_ERROR_CODE(0, 0x2612) #define TSDB_CODE_PAR_INVALID_ENDPOINT TAOS_DEF_ERROR_CODE(0, 0x2613) #define TSDB_CODE_PAR_EXPRIE_STATEMENT TAOS_DEF_ERROR_CODE(0, 0x2614) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 5837100bf2..9d28b63a15 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -75,6 +75,9 @@ fi if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : fi +if [ -f "${compile_dir}/test/cfg/taoskeeper.toml" ]; then + cp ${compile_dir}/test/cfg/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg || : +fi if [ -f "${compile_dir}/test/cfg/taoskeeper.service" ]; then cp ${compile_dir}/test/cfg/taoskeeper.service ${pkg_dir}${install_home_path}/cfg || : fi @@ -107,7 +110,7 @@ if [ -f "${compile_dir}/build/bin/taosadapter" ]; then cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: fi if [ -f "${compile_dir}/build/bin/taoskeeper" ]; then - cp ${compile_dir}/build/bin/taosadapter ${pkg_dir}${install_home_path}/bin ||: + cp ${compile_dir}/build/bin/taoskeeper ${pkg_dir}${install_home_path}/bin ||: fi if [ -f "${compile_dir}/../../../explorer/target/release/taos-explorer" ]; then diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index d67d436fa7..87f4f57fd3 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -282,5 +282,3 @@ else rm -rf ${install_dir} ||: # mv ../"$(basename ${pkg_name}).tar.gz" . fi - -cd ${curr_dir} diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 830144c12a..613645c4cd 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -68,10 +68,19 @@ static bool tmqAddJsonArrayItem(cJSON *array, cJSON *item){ } -static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen); -static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } +static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, uint32_t metaLen); +static tb_uid_t processSuid(tb_uid_t suid, char* db) { + if (db == NULL) { + return suid; + } + return suid + MurmurHash3_32(db, strlen(db)); +} static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, SColCmprWrapper* pColCmprRow, cJSON** pJson) { + if (schemaRow == NULL || name == NULL || pColCmprRow == NULL || pJson == NULL) { + uError("invalid parameter, schemaRow:%p, name:%p, pColCmprRow:%p, pJson:%p", schemaRow, name, pColCmprRow, pJson); + return; + } int32_t code = TSDB_CODE_SUCCESS; int8_t buildDefaultCompress = 0; if (pColCmprRow->nCols <= 0) { @@ -186,6 +195,9 @@ end: } static int32_t setCompressOption(cJSON* json, uint32_t para) { + if (json == NULL) { + return TSDB_CODE_INVALID_PARA; + } uint8_t encode = COMPRESS_L1_TYPE_U32(para); int32_t code = 0; if (encode != 0) { @@ -219,6 +231,10 @@ end: return code; } static void buildAlterSTableJson(void* alterData, int32_t alterDataLen, cJSON** pJson) { + if (alterData == NULL || pJson == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SMAlterStbReq req = {0}; cJSON* json = NULL; char* string = NULL; @@ -362,6 +378,10 @@ end: } static void processCreateStb(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (metaRsp == NULL || pJson == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SVCreateStbReq req = {0}; SDecoder coder; @@ -382,6 +402,10 @@ end: } static void processAlterStb(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (metaRsp == NULL || pJson == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SVCreateStbReq req = {0}; SDecoder coder = {0}; uDebug("alter stable data:%p", metaRsp); @@ -402,6 +426,10 @@ end: } static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { + if (json == NULL || pCreateReq == NULL) { + uError("invalid parameter in %s", __func__); + return; + } STag* pTag = (STag*)pCreateReq->ctb.pTag; char* sname = pCreateReq->ctb.stbName; char* name = pCreateReq->name; @@ -502,6 +530,10 @@ end: } static void buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs, cJSON** pJson) { + if (pJson == NULL || pCreateReq == NULL) { + uError("invalid parameter in %s", __func__); + return; + } int32_t code = 0; char* string = NULL; cJSON* json = cJSON_CreateObject(); @@ -531,6 +563,10 @@ end: } static void processCreateTable(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (pJson == NULL || metaRsp == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder decoder = {0}; SVCreateTbBatchReq req = {0}; SVCreateTbReq* pCreateReq; @@ -561,6 +597,10 @@ end: } static void processAutoCreateTable(SMqDataRsp* rsp, char** string) { + if (rsp == NULL || string == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder* decoder = NULL; SVCreateTbReq* pCreateReq = NULL; int32_t code = 0; @@ -611,6 +651,10 @@ end: } static void processAlterTable(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (pJson == NULL || metaRsp == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; @@ -850,6 +894,10 @@ end: } static void processDropSTable(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (pJson == NULL || metaRsp == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder decoder = {0}; SVDropStbReq req = {0}; cJSON* json = NULL; @@ -884,6 +932,10 @@ end: *pJson = json; } static void processDeleteTable(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (pJson == NULL || metaRsp == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDeleteRes req = {0}; SDecoder coder = {0}; cJSON* json = NULL; @@ -921,6 +973,10 @@ end: } static void processDropTable(SMqMetaRsp* metaRsp, cJSON** pJson) { + if (pJson == NULL || metaRsp == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; cJSON* json = NULL; @@ -958,7 +1014,11 @@ end: *pJson = json; } -static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosCreateStb(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SVCreateStbReq req = {0}; SDecoder coder; SMCreateStbReq pReq = {0}; @@ -973,8 +1033,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateStbReq(&coder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1068,7 +1128,11 @@ end: return code; } -static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosDropStb(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SVDropStbReq req = {0}; SDecoder coder = {0}; SMDropStbReq pReq = {0}; @@ -1083,8 +1147,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropStbReq(&coder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1173,11 +1237,19 @@ typedef struct SVgroupCreateTableBatch { } SVgroupCreateTableBatch; static void destroyCreateTbReqBatch(void* data) { + if (data == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosCreateTable(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SVCreateTbBatchReq req = {0}; SDecoder coder = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -1195,8 +1267,8 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1359,11 +1431,19 @@ typedef struct SVgroupDropTableBatch { } SVgroupDropTableBatch; static void destroyDropTbReqBatch(void* data) { + if (data == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosDropTable(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SVDropTbBatchReq req = {0}; SDecoder coder = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -1380,8 +1460,8 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1475,7 +1555,11 @@ end: return code; } -static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosDeleteData(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SDeleteRes req = {0}; SDecoder coder = {0}; char sql[256] = {0}; @@ -1484,8 +1568,8 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { uDebug("connId:0x%" PRIx64 " delete data, meta:%p, len:%d", *(int64_t*)taos, meta, metaLen); // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeDeleteRes(&coder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1510,7 +1594,11 @@ end: return code; } -static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t taosAlterTable(TAOS* taos, void* meta, uint32_t metaLen) { + if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SVAlterTbReq req = {0}; SDecoder dcoder = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -1527,8 +1615,8 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); - int32_t len = metaLen - sizeof(SMsgHead); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + uint32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&dcoder, data, len); if (tDecodeSVAlterTbReq(&dcoder, &req) < 0) { code = TSDB_CODE_INVALID_PARA; @@ -1632,7 +1720,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, int numFields, int64_t reqid) { - if (!taos || !pData || !tbname) { + if (taos == NULL || pData == NULL || tbname == NULL) { + uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } int32_t code = TSDB_CODE_SUCCESS; @@ -1693,7 +1782,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) } int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) { - if (!taos || !pData || !tbname) { + if (taos == NULL || pData == NULL || tbname == NULL) { return TSDB_CODE_INVALID_PARA; } int32_t code = TSDB_CODE_SUCCESS; @@ -1749,6 +1838,10 @@ end: } static void* getRawDataFromRes(void* pRetrieve) { + if (pRetrieve == NULL) { + uError("invalid parameter in %s", __func__); + return NULL; + } void* rawData = NULL; // deal with compatibility if (*(int64_t*)pRetrieve == 0) { @@ -1760,6 +1853,10 @@ static void* getRawDataFromRes(void* pRetrieve) { } static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) { + if (rsp == NULL || pHashObj == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } // find schema data info int32_t code = 0; SVCreateTbReq pCreateReq = {0}; @@ -1819,11 +1916,19 @@ typedef struct { } tbInfo; static void tmqFreeMeta(void* data) { + if (data == NULL) { + uError("invalid parameter in %s", __func__); + return; + } STableMeta* pTableMeta = *(STableMeta**)data; taosMemoryFree(pTableMeta); } static void freeRawCache(void* data) { + if (data == NULL) { + uError("invalid parameter in %s", __func__); + return; + } rawCacheInfo* pRawCache = (rawCacheInfo*)data; taosHashCleanup(pRawCache->pMetaHash); taosHashCleanup(pRawCache->pNameHash); @@ -1842,6 +1947,10 @@ static int32_t initRawCacheHash() { } static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) { + if (rawData == NULL || pTableMeta == NULL || pSW == NULL) { + uError("invalid parameter in %s", __func__); + return false; + } char* p = (char*)rawData; // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each // column length | @@ -1877,6 +1986,10 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe } static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** pMetaHash, void* key) { + if (pVgHash == NULL || pNameHash == NULL || pMetaHash == NULL || key == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; void* cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES); if (cacheInfo == NULL) { @@ -1905,6 +2018,10 @@ end: } static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pCatalog, SRequestConnInfo* conn) { + if (taos == NULL || pRequest == NULL || pCatalog == NULL || conn == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; RAW_RETURN_CHECK(buildRequest(*(int64_t*)taos, "", 0, NULL, false, pRequest, 0)); (*pRequest)->syncQuery = true; @@ -1924,26 +2041,38 @@ end: } typedef int32_t _raw_decode_func_(SDecoder* pDecoder, SMqDataRsp* pRsp); -static int32_t decodeRawData(SDecoder* decoder, void* data, int32_t dataLen, _raw_decode_func_ func, +static int32_t decodeRawData(SDecoder* decoder, void* data, uint32_t dataLen, _raw_decode_func_ func, SMqRspObj* rspObj) { - int8_t dataVersion = *(int8_t*)data; - if (dataVersion >= MQ_DATA_RSP_VERSION) { - data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); - dataLen -= sizeof(int8_t) + sizeof(int32_t); + if (decoder == NULL || data == NULL || func == NULL || rspObj == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } + int8_t dataVersion = *(int8_t*)data; + if (dataVersion >= MQ_DATA_RSP_VERSION) { + data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); + if (dataLen < sizeof(int8_t) + sizeof(int32_t)) { + return TSDB_CODE_INVALID_PARA; + } + dataLen -= sizeof(int8_t) + sizeof(int32_t); } - rspObj->resIter = -1; - tDecoderInit(decoder, data, dataLen); - int32_t code = func(decoder, &rspObj->dataRsp); - if (code != 0) { - SET_ERROR_MSG("decode mq taosx data rsp failed"); + rspObj->resIter = -1; + tDecoderInit(decoder, data, dataLen); + int32_t code = func(decoder, &rspObj->dataRsp); + if (code != 0) { + SET_ERROR_MSG("decode mq taosx data rsp failed"); } - return code; + return code; } static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj* pMetaHash, SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName, STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) { + if (pVgHash == NULL || pNameHash == NULL || pMetaHash == NULL || pCatalog == NULL || conn == NULL || pName == NULL || + pMeta == NULL || pSW == NULL || rawData == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; STableMeta* pTableMeta = NULL; tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname)); @@ -2000,7 +2129,11 @@ end: return code; } -static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { +static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) { + if (taos == NULL || data == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; SQuery* pQuery = NULL; SMqRspObj rspObj = {0}; @@ -2073,7 +2206,11 @@ end: return code; } -static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { +static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, uint32_t dataLen) { + if (taos == NULL || data == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } int32_t code = TSDB_CODE_SUCCESS; SQuery* pQuery = NULL; SMqRspObj rspObj = {0}; @@ -2162,6 +2299,10 @@ end: } static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) { + if (pMetaRsp == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); + return; + } if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) { processCreateStb(pMetaRsp, meta); } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_STB) { @@ -2182,6 +2323,10 @@ static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) { } static void processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp, char** string) { + if (pMsgRsp == NULL || string == NULL) { + uError("invalid parameter in %s", __func__); + return; + } SDecoder coder; SMqBatchMetaRsp rsp = {0}; int32_t code = 0; @@ -2228,7 +2373,10 @@ end: } char* tmq_get_json_meta(TAOS_RES* res) { - if (res == NULL) return NULL; + if (res == NULL) { + uError("invalid parameter in %s", __func__); + return NULL; + } uDebug("tmq_get_json_meta res:%p", res); if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res) && !TD_RES_TMQ_BATCH_META(res)) { return NULL; @@ -2256,6 +2404,10 @@ char* tmq_get_json_meta(TAOS_RES* res) { void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } static int32_t getOffSetLen(const SMqDataRsp* pRsp) { + if (pRsp == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } SEncoder coder = {0}; tEncoderInit(&coder, NULL, 0); if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; @@ -2267,44 +2419,48 @@ static int32_t getOffSetLen(const SMqDataRsp* pRsp) { typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp); static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) { - int32_t len = 0; - int32_t code = 0; - SEncoder encoder = {0}; - void* buf = NULL; - tEncodeSize(encodeFunc, rspObj, len, code); - if (code < 0) { - code = TSDB_CODE_INVALID_MSG; - goto FAILED; + if (raw == NULL || encodeFunc == NULL || rspObj == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; } - len += sizeof(int8_t) + sizeof(int32_t); - buf = taosMemoryCalloc(1, len); - if (buf == NULL) { - code = terrno; - goto FAILED; + uint32_t len = 0; + int32_t code = 0; + SEncoder encoder = {0}; + void* buf = NULL; + tEncodeSize(encodeFunc, rspObj, len, code); + if (code < 0) { + code = TSDB_CODE_INVALID_MSG; + goto FAILED; } - tEncoderInit(&encoder, buf, len); - if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto FAILED; + len += sizeof(int8_t) + sizeof(int32_t); + buf = taosMemoryCalloc(1, len); + if (buf == NULL) { + code = terrno; + goto FAILED; } - int32_t offsetLen = getOffSetLen(rspObj); - if (offsetLen <= 0) { - code = TSDB_CODE_INVALID_MSG; - goto FAILED; + tEncoderInit(&encoder, buf, len); + if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto FAILED; } - if (tEncodeI32(&encoder, offsetLen) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto FAILED; + int32_t offsetLen = getOffSetLen(rspObj); + if (offsetLen <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto FAILED; } - if (encodeFunc(&encoder, rspObj) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto FAILED; + if (tEncodeI32(&encoder, offsetLen) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto FAILED; } - tEncoderClear(&encoder); + if (encodeFunc(&encoder, rspObj) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto FAILED; + } + tEncoderClear(&encoder); - raw->raw = buf; - raw->raw_len = len; - return code; + raw->raw = buf; + raw->raw_len = len; + return code; FAILED: tEncoderClear(&encoder); taosMemoryFree(buf); @@ -2312,13 +2468,14 @@ FAILED: } int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { - if (!raw || !res) { + if (raw == NULL || res == NULL) { + uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } SMqRspObj* rspObj = ((SMqRspObj*)res); if (TD_RES_TMQ_META(res)) { raw->raw = rspObj->metaRsp.metaRsp; - raw->raw_len = rspObj->metaRsp.metaRspLen; + raw->raw_len = rspObj->metaRsp.metaRspLen >= 0 ? rspObj->metaRsp.metaRspLen : 0; raw->raw_type = rspObj->metaRsp.resMsgType; uDebug("tmq get raw type meta:%p", raw); } else if (TD_RES_TMQ(res)) { @@ -2378,6 +2535,10 @@ static int32_t writeRawInit() { } static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) { + if (taos == NULL || buf == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } if (writeRawInit() != 0) { return TSDB_CODE_INTERNAL_ERROR; } @@ -2415,8 +2576,9 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { return writeRawImpl(taos, raw.raw, raw.raw_len, raw.raw_type); } -static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen) { +static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, uint32_t metaLen) { if (taos == NULL || meta == NULL) { + uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } SMqBatchMetaRsp rsp = {0}; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b254f6eb8b..0cbdfc13e0 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -554,6 +554,9 @@ char** tmq_list_to_c_array(const tmq_list_t* list) { } static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { + if (pParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } int64_t refId = pParamSet->refId; int32_t code = 0; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); @@ -575,6 +578,9 @@ static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } static int32_t commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { + if (pParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, @@ -603,6 +609,9 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet) { + if (tmq == NULL || epSet == NULL || offset == NULL || pTopicName == NULL || pParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqVgOffset pOffset = {0}; pOffset.consumerId = tmq->consumerId; @@ -673,6 +682,9 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse } static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic** topic) { + if (tmq == NULL || pTopicName == NULL || topic == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < numOfTopics; ++i) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -689,6 +701,9 @@ static int32_t getTopicByName(tmq_t* tmq, const char* pTopicName, SMqClientTopic static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum, SMqCommitCbParamSet** ppParamSet) { + if (tmq == NULL || ppParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { return terrno; @@ -704,6 +719,9 @@ static int32_t prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, voi } static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) { + if (tmq == NULL || pTopicName == NULL || pVg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqClientTopic* pTopic = NULL; int32_t code = getTopicByName(tmq, pTopicName, &pTopic); if (code != 0) { @@ -724,6 +742,9 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient } static int32_t innerCommit(tmq_t* tmq, char* pTopicName, STqOffsetVal* offsetVal, SMqClientVg* pVg, SMqCommitCbParamSet* pParamSet){ + if (tmq == NULL || pTopicName == NULL || offsetVal == NULL || pVg == NULL || pParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; if (offsetVal->type <= 0) { code = TSDB_CODE_TMQ_INVALID_MSG; @@ -754,6 +775,9 @@ static int32_t innerCommit(tmq_t* tmq, char* pTopicName, STqOffsetVal* offsetVal static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, tmq_commit_cb* pCommitFp, void* userParam) { + if (tmq == NULL || pTopicName == NULL || offsetVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } tqInfoC("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); SMqCommitCbParamSet* pParamSet = NULL; int32_t code = prepareCommitCbParamSet(tmq, pCommitFp, userParam, 0, &pParamSet); @@ -807,6 +831,9 @@ end: } static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){ + if (tmq == NULL || pParamSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; taosRLockLatch(&tmq->lock); int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); @@ -842,6 +869,9 @@ END: } static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam) { + if (tmq == NULL) { + return; + } int32_t code = 0; SMqCommitCbParamSet* pParamSet = NULL; // init waitingRspNum as DEFAULT_COMMIT_CNT to prevent concurrency issue @@ -1071,12 +1101,15 @@ END: } static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) { - if (code != 0) { + if (code != 0 && pTmq != NULL) { tqErrorC("consumer:0x%" PRIx64 ", failed to commit offset, code:%s", pTmq->consumerId, tstrerror(code)); } } static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { + if (rspWrapper == NULL) { + return; + } if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { tDeleteSMqAskEpRsp(&rspWrapper->epRsp); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { @@ -1091,12 +1124,18 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { } static void freeClientVg(void* param) { + if (param == NULL) { + return; + } SMqClientVg* pVg = param; tOffsetDestroy(&pVg->offsetInfo.endOffset); tOffsetDestroy(&pVg->offsetInfo.beginOffset); tOffsetDestroy(&pVg->offsetInfo.committedOffset); } static void freeClientTopic(void* param) { + if (param == NULL) { + return; + } SMqClientTopic* pTopic = param; taosMemoryFreeClear(pTopic->schema.pSchema); taosArrayDestroyEx(pTopic->vgs, freeClientVg); @@ -1104,6 +1143,9 @@ static void freeClientTopic(void* param) { static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, tmq_t* tmq) { + if (pTopic == NULL || pTopicEp == NULL || pVgOffsetHashMap == NULL || tmq == NULL) { + return; + } pTopic->schema = pTopicEp->schema; pTopicEp->schema.nCols = 0; pTopicEp->schema.pSchema = NULL; @@ -1167,6 +1209,9 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic } static void buildNewTopicList(tmq_t* tmq, SArray* newTopics, const SMqAskEpRsp* pRsp){ + if (tmq == NULL || newTopics == NULL || pRsp == NULL) { + return; + } SHashObj* pVgOffsetHashMap = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); if (pVgOffsetHashMap == NULL) { tqErrorC("consumer:0x%" PRIx64 " taos hash init null, code:%d", tmq->consumerId, terrno); @@ -1221,6 +1266,9 @@ static void buildNewTopicList(tmq_t* tmq, SArray* newTopics, const SMqAskEpRsp* } static void doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { + if (tmq == NULL || pRsp == NULL) { + return; + } int32_t topicNumGet = taosArrayGetSize(pRsp->topics); // vnode transform (epoch == tmq->epoch && topicNumGet != 0) // ask ep rsp (epoch == tmq->epoch && topicNumGet == 0) @@ -1337,6 +1385,9 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { } static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { + if (pTmq == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqAskEpReq req = {0}; req.consumerId = pTmq->consumerId; req.epoch = updateEpSet ? -1 : pTmq->epoch; @@ -1395,6 +1446,9 @@ static int32_t askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { } void tmqHandleAllDelayedTask(tmq_t* pTmq) { + if (pTmq == NULL) { + return; + } STaosQall* qall = NULL; int32_t code = 0; @@ -1443,6 +1497,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) { } void tmqClearUnhandleMsg(tmq_t* tmq) { + if (tmq == NULL) return; SMqRspWrapper* rspWrapper = NULL; while (taosGetQitem(tmq->qall, (void**)&rspWrapper) != 0) { tmqFreeRspWrapper(rspWrapper); @@ -1507,6 +1562,7 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { } void tmqFreeImpl(void* handle) { + if (handle == NULL) return; tmq_t* tmq = (tmq_t*)handle; int64_t id = tmq->consumerId; @@ -1715,6 +1771,7 @@ _failed: } static int32_t syncAskEp(tmq_t* pTmq) { + if (pTmq == NULL) return TSDB_CODE_INVALID_PARA; SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); if (pInfo == NULL) return terrno; if (tsem2_init(&pInfo->sem, 0, 0) != 0) { @@ -1897,6 +1954,9 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para } static void getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId, SMqClientVg** pVg) { + if (tmq == NULL || topicName == NULL || pVg == NULL) { + return; + } int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int i = 0; i < topicNumCur; i++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); @@ -1914,6 +1974,9 @@ static void getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId, SMqClientVg** p } static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { + if (tmq == NULL || topicName == NULL) { + return NULL; + } int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int i = 0; i < topicNumCur; i++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); @@ -2026,6 +2089,9 @@ EXIT: } void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { + if (pReq == NULL || tmq == NULL || pTopic == NULL || pVg == NULL) { + return; + } (void)snprintf(pReq->subKey, TSDB_SUBSCRIBE_KEY_LEN, "%s%s%s", tmq->groupId, TMQ_SEPARATOR, pTopic->topicName); pReq->withTbName = tmq->withTbName; pReq->consumerId = tmq->consumerId; @@ -2072,7 +2138,7 @@ void changeByteEndian(char* pData) { } static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, int64_t* rows, int32_t* precision) { - if (pRetrieve == NULL) { + if (pRetrieve == NULL || rawData == NULL || rows == NULL) { return; } if (*(int64_t*)pRetrieve == 0) { @@ -2092,6 +2158,9 @@ static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, i static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObj* pRspObj) { + if (pWrapper == NULL || pVg == NULL || numOfRows == NULL || pRspObj == NULL) { + return; + } pRspObj->resIter = -1; pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -2130,6 +2199,9 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { + if (pTmq == NULL || pTopic == NULL || pVg == NULL) { + return TSDB_CODE_INVALID_MSG; + } SMqPollReq req = {0}; char* msg = NULL; SMqPollCbParam* pParam = NULL; @@ -2199,6 +2271,9 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { + if (tmq == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; taosWLockLatch(&tmq->lock); @@ -2258,6 +2333,9 @@ end: static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData) { + if (pVg == NULL || reqOffset == NULL || rspOffset == NULL) { + return; + } if (!pVg->seekUpdated) { tqDebugC("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); if (hasData) { @@ -2283,6 +2361,9 @@ static SMqRspObj* buildRsp(SMqPollRspWrapper* pollRspWrapper){ SMqBatchMetaRsp batchMetaRsp; } MEMSIZE; + if (pollRspWrapper == NULL) { + return NULL; + } SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); if (pRspObj == NULL) { tqErrorC("buildRsp:failed to allocate memory"); @@ -2297,6 +2378,9 @@ static SMqRspObj* buildRsp(SMqPollRspWrapper* pollRspWrapper){ } static void processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + if (tmq == NULL || pRspWrapper == NULL) { + return; + } SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform @@ -2322,6 +2406,9 @@ static void processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ taosWUnLockLatch(&tmq->lock); } static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ + if (tmq == NULL || pRspWrapper == NULL) { + return NULL; + } SMqRspObj* pRspObj = NULL; if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { @@ -2401,6 +2488,9 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ } static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { + if (tmq == NULL) { + return NULL; + } tqDebugC("consumer:0x%" PRIx64 " start to handle the rsp, total:%d", tmq->consumerId, taosQallItemSize(tmq->qall)); void* returnVal = NULL; @@ -2478,6 +2568,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } static void displayConsumeStatistics(tmq_t* pTmq) { + if (pTmq == NULL) return; taosRLockLatch(&pTmq->lock); int32_t numOfTopics = taosArrayGetSize(pTmq->clientTopics); tqInfoC("consumer:0x%" PRIx64 " closing poll:%" PRId64 " rows:%" PRId64 " topics:%d, final epoch:%d", @@ -2680,7 +2771,11 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* } } -static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param) { +static void commitCallBackFn(tmq_t* tmq, int32_t code, void* param) { + if (param == NULL) { + tqErrorC("invalid param in commit cb"); + return; + } SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; if (tsem2_post(&pInfo->sem) != 0){ @@ -2732,6 +2827,10 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { // wal range will be ok after calling tmq_get_topic_assignment or poll interface static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { + if (offset == NULL) { + tqErrorC("invalid offset, null"); + return TSDB_CODE_INVALID_PARA; + } if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { tqErrorC("Assignment or poll interface need to be called first"); return TSDB_CODE_TMQ_NEED_INITIALIZED; @@ -2852,6 +2951,9 @@ end: int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pResInfo) { + if (res == NULL || pResInfo == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqRspObj* pRspObj = (SMqRspObj*)res; SMqDataRsp* data = &pRspObj->dataRsp; @@ -2889,7 +2991,7 @@ int32_t tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4, SReqResultInfo** pRes } static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { - if (param == NULL) { + if (param == NULL || pMsg == NULL) { return code; } SMqVgWalInfoParam* pParam = param; @@ -2962,6 +3064,9 @@ static bool isInSnapshotMode(int8_t type, bool useSnapshot) { } static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { + if (param == NULL) { + return code; + } SMqCommittedParam* pParam = param; if (code != 0) { @@ -2992,6 +3097,9 @@ end: } int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* epSet) { + if (tmq == NULL || tname == NULL || epSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMqVgOffset pOffset = {0}; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 905dcb4fda..8a8fce8f7e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "cJSON.h" #include "defines.h" #include "os.h" #include "osString.h" #include "tconfig.h" -#include "tglobal.h" #include "tgrant.h" #include "tjson.h" #include "tlog.h" @@ -104,6 +104,7 @@ int32_t tsRetentionSpeedLimitMB = 0; // unlimited const char *tsAlterCompactTaskKeywords = "max_compact_tasks"; int32_t tsNumOfCompactThreads = 2; +int32_t tsNumOfRetentionThreads = 1; // sync raft int32_t tsElectInterval = 25 * 1000; @@ -328,6 +329,7 @@ int64_t tsStreamBufferSize = 128 * 1024 * 1024; bool tsFilterScalarMode = false; int tsResolveFQDNRetryTime = 100; // seconds int tsStreamAggCnt = 100000; +bool tsStreamCoverage = false; bool tsUpdateCacheBatch = true; @@ -733,6 +735,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + + TAOS_CHECK_RETURN(cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL)); + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -1463,6 +1468,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "bypassFlag"); tsBypassFlag = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamCoverage"); + tsStreamCoverage = pItem->bval; + TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -2735,7 +2743,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, const char *name) { {"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}, {"tsmaDataDeleteMark", &tsmaDataDeleteMark}, {"numOfRpcSessions", &tsNumOfRpcSessions}, - {"bypassFlag", &tsBypassFlag}}; + {"bypassFlag", &tsBypassFlag}, + {"streamCoverage", &tsStreamCoverage}}; if ((code = taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true)) != TSDB_CODE_SUCCESS) { code = taosCfgSetOption(options, tListLen(options), pItem, false); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 2108a097ee..bfe4cd165e 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -69,6 +69,7 @@ int32_t dmStartStatusThread(SDnodeMgmt *pMgmt); int32_t dmStartConfigThread(SDnodeMgmt *pMgmt); int32_t dmStartStatusInfoThread(SDnodeMgmt *pMgmt); void dmStopStatusThread(SDnodeMgmt *pMgmt); +void dmStopConfigThread(SDnodeMgmt *pMgmt); void dmStopStatusInfoThread(SDnodeMgmt *pMgmt); int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt); void dmStopNotifyThread(SDnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index ccc6439b5d..625cca7087 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -475,7 +475,7 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } -extern void tsdbAlterMaxCompactTasks(); +extern void tsdbAlterNumCompactThreads(); static int32_t dmAlterMaxCompactTask(const char *value) { int32_t max_compact_tasks; char *endptr = NULL; @@ -489,7 +489,7 @@ static int32_t dmAlterMaxCompactTask(const char *value) { dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks); tsNumOfCompactThreads = max_compact_tasks; #ifdef TD_ENTERPRISE - tsdbAlterMaxCompactTasks(); + (void)tsdbAlterNumCompactThreads(); #endif } diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index b58c1a216d..b3b1df314a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -52,6 +52,7 @@ static void dmStopMgmt(SDnodeMgmt *pMgmt) { dmStopMonitorThread(pMgmt); dmStopAuditThread(pMgmt); dmStopStatusThread(pMgmt); + dmStopConfigThread(pMgmt); dmStopStatusInfoThread(pMgmt); #if defined(TD_ENTERPRISE) dmStopNotifyThread(pMgmt); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 8f890f6805..ef4e76031d 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -343,7 +343,7 @@ int32_t dmStartConfigThread(SDnodeMgmt *pMgmt) { int32_t code = 0; TdThreadAttr thAttr; (void)taosThreadAttrInit(&thAttr); - (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); if (taosThreadCreate(&pMgmt->configThread, &thAttr, dmConfigThreadFp, pMgmt) != 0) { code = TAOS_SYSTEM_ERROR(errno); dError("failed to create config thread since %s", tstrerror(code)); @@ -378,6 +378,13 @@ void dmStopStatusThread(SDnodeMgmt *pMgmt) { } } +void dmStopConfigThread(SDnodeMgmt *pMgmt) { + if (taosCheckPthreadValid(pMgmt->configThread)) { + (void)taosThreadJoin(pMgmt->configThread, NULL); + taosThreadClear(&pMgmt->configThread); + } +} + void dmStopStatusInfoThread(SDnodeMgmt *pMgmt) { if (taosCheckPthreadValid(pMgmt->statusInfoThread)) { (void)taosThreadJoin(pMgmt->statusInfoThread, NULL); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 537c1f6297..6d5e117181 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -896,7 +896,7 @@ static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("vnode-sync", "initialized"); - if ((code = vnodeInit(tsNumOfCommitThreads, pInput->stopDnodeFp)) != 0) { + if ((code = vnodeInit(pInput->stopDnodeFp)) != 0) { dError("failed to init vnode since %s", tstrerror(code)); goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index fbce8f544a..6e9dc6ab17 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -52,6 +52,9 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete, }; + if (pMnode == NULL){ + return TSDB_CODE_INVALID_PARA; + } mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); @@ -66,6 +69,9 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} int32_t mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SRpcHandleInfo *info) { + if (pMnode == NULL || info == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; void *msg = rpcMallocCont(sizeof(int64_t)); MND_TMQ_NULL_CHECK(msg); @@ -88,6 +94,9 @@ END: } static int32_t validateTopics(STrans* pTrans, SCMSubscribeReq *subscribe, SMnode *pMnode, const char *pUser) { + if (pTrans == NULL || subscribe == NULL || pMnode == NULL || pUser == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqTopicObj *pTopic = NULL; int32_t code = 0; @@ -130,6 +139,9 @@ END: } static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMnode *pMnode = pMsg->info.node; SMqConsumerClearMsg *pClearMsg = pMsg->pCont; @@ -155,6 +167,9 @@ END: } static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { + if (pMnode == NULL || pConsumer == NULL || rsp == NULL || user == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege)); MND_TMQ_NULL_CHECK(rsp->topicPrivileges); @@ -181,6 +196,9 @@ END: } static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pConsumer){ + if (pMnode == NULL || req == NULL || pConsumer == NULL){ + return; + } for (int i = 0; i < taosArrayGetSize(req->topics); i++) { TopicOffsetRows *data = taosArrayGet(req->topics, i); if (data == NULL){ @@ -210,6 +228,9 @@ static void storeOffsetRows(SMnode *pMnode, SMqHbReq *req, SMqConsumerObj *pCons } static int32_t buildMqHbRsp(SRpcMsg *pMsg, SMqHbRsp *rsp){ + if (pMsg == NULL || rsp == NULL){ + return TSDB_CODE_INVALID_PARA; + } int32_t tlen = tSerializeSMqHbRsp(NULL, 0, rsp); if (tlen <= 0){ return TSDB_CODE_TMQ_INVALID_MSG; @@ -229,6 +250,9 @@ static int32_t buildMqHbRsp(SRpcMsg *pMsg, SMqHbRsp *rsp){ } static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMnode *pMnode = pMsg->info.node; SMqHbReq req = {0}; @@ -256,6 +280,9 @@ END: } static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){ + if (pMnode == NULL || pConsumer == NULL || rsp == NULL){ + return TSDB_CODE_INVALID_PARA; + } taosRLockLatch(&pConsumer->lock); int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); @@ -359,6 +386,9 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e } static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){ + if (pMsg == NULL || rsp == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; // encode rsp int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp); @@ -388,6 +418,9 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc } static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pMsg->info.node; SMqAskEpReq req = {0}; SMqAskEpRsp rsp = {0}; @@ -431,6 +464,9 @@ END: } int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { + if (pConsumer == NULL || pTrans == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); MND_TMQ_NULL_CHECK(pCommitRaw); @@ -445,6 +481,9 @@ END: } int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { + if (pConsumer == NULL || pTrans == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); MND_TMQ_NULL_CHECK(pCommitRaw); @@ -459,6 +498,9 @@ END: } static void freeItem(void *param) { + if (param == NULL) { + return; + } void *pItem = *(void **)param; if (pItem != NULL) { taosMemoryFree(pItem); @@ -475,6 +517,9 @@ if (taosArrayPush(pConsumerNew->array, &newTopicCopy) == NULL){\ } static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerObj *pConsumerNew){ + if (pExistedConsumer == NULL || pConsumerNew == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; pConsumerNew->rebNewTopics = taosArrayInit(0, sizeof(void *)); MND_TMQ_NULL_CHECK(pConsumerNew->rebNewTopics); @@ -528,6 +573,9 @@ END: } static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ + if (pTopicList == NULL || pMnode == NULL) { + return TSDB_CODE_INVALID_PARA; + } taosArraySort(pTopicList, taosArrayCompareString); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); @@ -542,6 +590,9 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ } static int32_t buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe, SMqConsumerObj** ppConsumer){ + if (pMnode == NULL || subscribe == NULL) { + return TSDB_CODE_INVALID_PARA; + } int64_t consumerId = subscribe->consumerId; char *cgroup = subscribe->cgroup; SMqConsumerObj *pConsumerNew = NULL; @@ -581,6 +632,9 @@ END: } int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pMsg->info.node; char *msgStr = pMsg->pCont; int32_t code = 0; @@ -619,6 +673,9 @@ END: } SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { + if (pConsumer == NULL) { + return NULL; + } int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -659,6 +716,9 @@ CM_ENCODE_OVER: } SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { + if (pRaw == NULL) { + return NULL; + } int32_t code = 0; int32_t lino = 0; SSdbRow *pRow = NULL; @@ -717,6 +777,9 @@ CM_DECODE_OVER: } static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { + if (pConsumer == NULL) { + return TSDB_CODE_INVALID_PARA; + } mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d", pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status), pConsumer->epoch); pConsumer->subscribeTime = pConsumer->createTime; @@ -724,6 +787,9 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { + if (pConsumer == NULL) { + return TSDB_CODE_INVALID_PARA; + } mInfo("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status, mndConsumerStatusName(pConsumer->status)); tClearSMqConsumerObj(pConsumer); @@ -744,6 +810,9 @@ static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { // remove from topic list static void removeFromTopicList(SArray *topicList, const char *pTopic, int64_t consumerId, char *type) { + if (topicList == NULL || pTopic == NULL) { + return; + } int32_t size = taosArrayGetSize(topicList); for (int32_t i = 0; i < size; i++) { char *p = taosArrayGetP(topicList, i); @@ -759,6 +828,9 @@ static void removeFromTopicList(SArray *topicList, const char *pTopic, int64_t c } static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char *pTopic) { + if (pConsumer == NULL || pTopic == NULL) { + return false; + } bool existing = false; int32_t size = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < size; i++) { @@ -773,6 +845,9 @@ static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { + if (pOldConsumer == NULL || pNewConsumer == NULL) { + return TSDB_CODE_INVALID_PARA; + } mInfo("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", createTime:%" PRId64, pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime); @@ -857,6 +932,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** pConsumer) { + if (pMnode == NULL || pConsumer == NULL) { + return TSDB_CODE_INVALID_PARA; + } SSdb *pSdb = pMnode->pSdb; *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); if (*pConsumer == NULL) { @@ -866,11 +944,17 @@ int32_t mndAcquireConsumer(SMnode *pMnode, int64_t consumerId, SMqConsumerObj** } void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) { + if (pMnode == NULL || pConsumer == NULL) { + return; + } SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pConsumer); } static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { + if (pReq == NULL || pShow == NULL || pBlock == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -1021,6 +1105,7 @@ END: } static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { + if (pMnode == NULL || pIter == NULL) return; SSdb *pSdb = pMnode->pSdb; sdbCancelFetchByType(pSdb, pIter, SDB_CONSUMER); } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index bcca01a230..e5ab02996a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -43,6 +43,9 @@ static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { + if (pTrans == NULL || pSub == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); MND_TMQ_NULL_CHECK(pCommitRaw); @@ -68,6 +71,9 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete, }; + if (pMnode == NULL) { + return TSDB_CODE_INVALID_PARA; + } mndSetMsgHandle(pMnode, TDMT_VND_TMQ_SUBSCRIBE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DELETE_SUB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessRebalanceReq); @@ -81,6 +87,9 @@ int32_t mndInitSubscribe(SMnode *pMnode) { } static int32_t mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *subKey, SMqSubscribeObj** pSub) { + if(pMnode == NULL || pTopic == NULL || subKey == NULL || pSub == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; MND_TMQ_RETURN_CHECK(tNewSubscribeObj(subKey, pSub)); (*pSub)->dbUid = pTopic->dbUid; @@ -99,6 +108,9 @@ END: static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + if (pSub == NULL || pRebVg == NULL || pBuf == NULL || pLen == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqRebVgReq req = {0}; int32_t code = 0; SEncoder encoder = {0}; @@ -146,6 +158,9 @@ END: static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { + if (pMnode == NULL || pTrans == NULL || pSub == NULL || pRebVg == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; void *buf = NULL; @@ -180,6 +195,9 @@ END: } static void mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, bool fullName) { + if (key == NULL || topic == NULL || cgroup == NULL) { + return; + } int32_t i = 0; while (key[i] != TMQ_SEPARATOR_CHAR) { i++; @@ -197,6 +215,9 @@ static void mndSplitSubscribeKey(const char *key, char *topic, char *cgroup, boo } static int32_t mndGetOrCreateRebSub(SHashObj *pHash, const char *key, SMqRebInfo **pReb) { + if (pHash == NULL || key == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMqRebInfo* pRebInfo = taosHashGet(pHash, key, strlen(key) + 1); if (pRebInfo == NULL) { @@ -222,6 +243,9 @@ END: } static int32_t pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) { + if (vgs == NULL || pHash == NULL || key == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMqVgEp **pVgEp = (SMqVgEp **)taosArrayPop(vgs); MND_TMQ_NULL_CHECK(pVgEp); @@ -233,6 +257,9 @@ END: } static int32_t processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { + if (pHash == NULL || pOutput == NULL || pInput == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; @@ -266,6 +293,9 @@ END: } static int32_t processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { + if (pOutput == NULL || pInput == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); @@ -285,6 +315,9 @@ END: } static int32_t processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { + if (pOutput == NULL || pHash == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); for (int32_t i = 0; i < numOfVgroups; i++) { @@ -296,6 +329,9 @@ END: static int32_t processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, int32_t remainderVgCnt) { + if (pOutput == NULL || pHash == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t cnt = 0; void *pIter = NULL; @@ -328,6 +364,9 @@ END: } static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { + if (pMnode == NULL || pOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t totalVgNum = 0; SVgObj *pVgroup = NULL; @@ -403,6 +442,9 @@ END: } static int32_t processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + if (pMnode == NULL || pInput == NULL || pOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqSubscribeObj *pSub = NULL; int32_t code = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key, &pSub); // put all offset rows if( code != 0){ @@ -465,6 +507,7 @@ END: } static void printRebalanceLog(SMqRebOutputObj *pOutput) { + if (pOutput == NULL) return; mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); @@ -492,6 +535,9 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput) { static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, int32_t *minVgCnt, int32_t *remainderVgCnt) { + if (pInput == NULL || pSubKey == NULL || minVgCnt == NULL || remainderVgCnt == NULL) { + return; + } int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers); int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved; @@ -509,6 +555,9 @@ static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, con } static int32_t assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) { + if (pOutput == NULL || pHash == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqRebOutputVg *pRebVg = NULL; void *pAssignIter = NULL; void *pIter = NULL; @@ -580,6 +629,9 @@ END: } static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { + if (pMnode == NULL || pInput == NULL || pOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput); if (totalVgNum < 0){ return totalVgNum; @@ -605,6 +657,9 @@ END: } static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t type, char *cgroup, char *topic) { + if (pTrans == NULL || consumers == NULL || cgroup == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMqConsumerObj *pConsumerNew = NULL; int32_t consumerNum = taosArrayGetSize(consumers); @@ -623,6 +678,9 @@ END: } static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic) { + if (pTrans == NULL || pOutput == NULL || cgroup == NULL || topic == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; MND_TMQ_RETURN_CHECK(presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL)); MND_TMQ_RETURN_CHECK(presistConsumerByType(pTrans, pOutput->newConsumers, CONSUMER_ADD_REB, cgroup, topic)); @@ -632,6 +690,9 @@ END: } static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { + if (pMnode == NULL || pMsg == NULL || pOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } struct SSubplan *pPlan = NULL; int32_t code = 0; STrans *pTrans = NULL; @@ -682,6 +743,7 @@ END: } static void freeRebalanceItem(void *param) { + if (param == NULL) return; SMqRebInfo *pInfo = param; taosArrayDestroy(pInfo->newConsumers); taosArrayDestroy(pInfo->removedConsumers); @@ -689,6 +751,9 @@ static void freeRebalanceItem(void *param) { // type = 0 remove type = 1 add static int32_t buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) { + if (rebSubHash == NULL || topicList == NULL || group == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t topicNum = taosArrayGetSize(topicList); for (int32_t i = 0; i < topicNum; i++) { @@ -709,6 +774,9 @@ END: } static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj *rebSubHash) { + if (pMnode == NULL || pConsumer == NULL || rebSubHash == NULL) { + return; + } int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < newTopicNum; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); @@ -754,6 +822,9 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash } static int32_t mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) { + if (pMsg == NULL || rebSubHash == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = NULL; @@ -818,6 +889,9 @@ void mndRebCntDec() { } static void clearRebOutput(SMqRebOutputObj *rebOutput) { + if (rebOutput == NULL) { + return; + } taosArrayDestroy(rebOutput->newConsumers); taosArrayDestroy(rebOutput->modifyConsumers); taosArrayDestroy(rebOutput->removedConsumers); @@ -827,6 +901,9 @@ static void clearRebOutput(SMqRebOutputObj *rebOutput) { } static int32_t initRebOutput(SMqRebOutputObj *rebOutput) { + if (rebOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; rebOutput->newConsumers = taosArrayInit(0, sizeof(int64_t)); MND_TMQ_NULL_CHECK(rebOutput->newConsumers); @@ -845,6 +922,9 @@ END: // This function only works when there are dirty consumers static int32_t checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { + if (pMnode == NULL || pSub == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; void *pIter = NULL; while (1) { @@ -871,6 +951,9 @@ END: } static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + if (pMnode == NULL || rebInput == NULL || rebOutput == NULL) { + return TSDB_CODE_INVALID_PARA; + } const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = NULL; int32_t code = mndAcquireSubscribeByKey(pMnode, key, &pSub); @@ -922,6 +1005,9 @@ END: } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } int code = 0; void *pIter = NULL; SMnode *pMnode = pMsg->info.node; @@ -986,6 +1072,9 @@ END: } static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { + if (pMnode == NULL || pSub == NULL || pTrans == NULL) { + return TSDB_CODE_INVALID_PARA; + } void *pIter = NULL; SVgObj *pVgObj = NULL; int32_t code = 0; @@ -1024,6 +1113,9 @@ END: } static int32_t mndCheckConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { + if (pMnode == NULL || pTrans == NULL || cgroup == NULL || topic == NULL) { + return TSDB_CODE_INVALID_PARA; + } void *pIter = NULL; SMqConsumerObj *pConsumer = NULL; int code = 0; @@ -1056,6 +1148,9 @@ END: } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { + if (pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pMsg->info.node; SMDropCgroupReq dropReq = {0}; STrans *pTrans = NULL; @@ -1109,6 +1204,9 @@ END: void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { + if (pSub == NULL) { + return NULL; + } int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1149,6 +1247,9 @@ SUB_ENCODE_OVER: } static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { + if (pRaw == NULL) { + return NULL; + } int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1218,17 +1319,18 @@ SUB_DECODE_OVER: } static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { - mTrace("subscribe:%s, perform insert action", pSub->key); + mTrace("subscribe:%s, perform insert action", pSub != NULL ? pSub->key : "null"); return 0; } static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { - mTrace("subscribe:%s, perform delete action", pSub->key); + mTrace("subscribe:%s, perform delete action", pSub != NULL ? pSub->key : "null"); tDeleteSubscribeObj(pSub); return 0; } static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub) { + if (pOldSub == NULL || pNewSub == NULL) return -1; mTrace("subscribe:%s, perform update action", pOldSub->key); taosWLockLatch(&pOldSub->lock); @@ -1249,6 +1351,9 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc } int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeObj** pSub) { + if (pMnode == NULL || key == NULL || pSub == NULL){ + return TSDB_CODE_INVALID_PARA; + } SSdb *pSdb = pMnode->pSdb; *pSub = sdbAcquire(pSdb, SDB_SUBSCRIBE, key); if (*pSub == NULL) { @@ -1258,6 +1363,7 @@ int32_t mndAcquireSubscribeByKey(SMnode *pMnode, const char *key, SMqSubscribeOb } int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) { + if (pMnode == NULL || topicName == NULL) return 0; int32_t num = 0; SSdb *pSdb = pMnode->pSdb; @@ -1283,11 +1389,13 @@ int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) { } void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { + if (pMnode == NULL || pSub == NULL) return; SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pSub); } int32_t mndSetDropSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { + if (pMnode == NULL || pTrans == NULL || pSub == NULL) return TSDB_CODE_INVALID_PARA; int32_t code = 0; SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); MND_TMQ_NULL_CHECK(pCommitRaw); @@ -1302,6 +1410,7 @@ END: } int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) { + if (pMnode == NULL || pTrans == NULL || topicName == NULL) return TSDB_CODE_INVALID_PARA; SSdb *pSdb = pMnode->pSdb; int32_t code = 0; void *pIter = NULL; @@ -1337,6 +1446,9 @@ END: static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char* user, const char* fqdn, const char *topic, const char *cgroup, SArray *vgs, SArray *offsetRows) { + if (pBlock == NULL || numOfRows == NULL || topic == NULL || cgroup == NULL){ + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t sz = taosArrayGetSize(vgs); for (int32_t j = 0; j < sz; j++) { @@ -1424,6 +1536,9 @@ END: } int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { + if (pReq == NULL || pShow == NULL || pBlock == NULL){ + return TSDB_CODE_INVALID_PARA; + } SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -1485,10 +1600,16 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock return numOfRows; END: + taosRUnLockLatch(&pSub->lock); + sdbRelease(pSdb, pSub); + return code; } void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter) { + if (pMnode == NULL) { + return; + } SSdb *pSdb = pMnode->pSdb; sdbCancelFetchByType(pSdb, pIter, SDB_SUBSCRIBE); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 643bab568f..5c199eddbd 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -55,6 +55,9 @@ int32_t mndInitTopic(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTopicActionDelete, }; + if (pMnode == NULL) { + return TSDB_CODE_INVALID_PARA; + } mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CREATE_TOPIC, mndProcessCreateTopicReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_TOPIC, mndProcessDropTopicReq); mndSetMsgHandle(pMnode, TDMT_VND_TMQ_ADD_CHECKINFO_RSP, mndTransProcessRsp); @@ -81,6 +84,9 @@ void mndTopicGetShowName(const char* fullTopic, char* topic) { } SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { + if (pTopic == NULL) { + return NULL; + } int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -172,6 +178,7 @@ TOPIC_ENCODE_OVER: } SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { + if (pRaw == NULL) return NULL; int32_t code = 0; int32_t lino = 0; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -193,7 +200,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { pTopic = sdbGetRowObj(pRow); if (pTopic == NULL) goto TOPIC_DECODE_OVER; - int32_t len; + int32_t len = 0; int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER); @@ -292,11 +299,12 @@ TOPIC_DECODE_OVER: } static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) { - mTrace("topic:%s perform insert action", pTopic->name); + mTrace("topic:%s perform insert action", pTopic != NULL ? pTopic->name : "null"); return 0; } static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { + if (pTopic == NULL) return 0; mTrace("topic:%s perform delete action", pTopic->name); taosMemoryFreeClear(pTopic->sql); taosMemoryFreeClear(pTopic->ast); @@ -307,6 +315,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { } static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { + if (pOldTopic == NULL || pNewTopic == NULL) return 0; mTrace("topic:%s perform update action", pOldTopic->name); (void)atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); (void)atomic_exchange_32(&pOldTopic->version, pNewTopic->version); @@ -315,6 +324,9 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic } int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTopic) { + if (pMnode == NULL || topicName == NULL || pTopic == NULL){ + return TSDB_CODE_INVALID_PARA; + } SSdb *pSdb = pMnode->pSdb; *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); if (*pTopic == NULL) { @@ -324,11 +336,13 @@ int32_t mndAcquireTopic(SMnode *pMnode, const char *topicName, SMqTopicObj **pTo } void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { + if (pMnode == NULL) return; SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pTopic); } static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { + if (pCreate == NULL) return TSDB_CODE_INVALID_PARA; if (pCreate->sql == NULL) return TSDB_CODE_MND_INVALID_TOPIC; if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { @@ -343,6 +357,7 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { } static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { + if (pAst == NULL || pTopic == NULL) return TSDB_CODE_INVALID_PARA; SNodeList *pNodeList = NULL; int32_t code = 0; MND_TMQ_RETURN_CHECK(nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList)); @@ -367,6 +382,7 @@ END: } static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj *topicObj){ + if (pTrans == NULL || pMnode == NULL || topicObj == NULL) return TSDB_CODE_INVALID_PARA; STqCheckInfo info = {0}; (void)memcpy(info.topic, topicObj->name, TSDB_TOPIC_FNAME_LEN); info.ntbUid = topicObj->ntbUid; @@ -388,7 +404,7 @@ static int32_t sendCheckInfoToVnode(STrans *pTrans, SMnode *pMnode, SMqTopicObj } // encoder check alter info - int32_t len; + int32_t len = 0; tEncodeSize(tEncodeSTqCheckInfo, &info, len, code); if (code != 0) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -426,6 +442,7 @@ END: static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb, const char *userName) { + if (pMnode == NULL || pReq == NULL || pCreate == NULL || pDb == NULL || userName == NULL) return TSDB_CODE_INVALID_PARA; mInfo("start to create topic:%s", pCreate->name); STrans *pTrans = NULL; int32_t code = 0; @@ -519,6 +536,9 @@ END: } static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { + if (pReq == NULL || pReq->contLen <= 0) { + return TSDB_CODE_INVALID_MSG; + } SMnode *pMnode = pReq->info.node; int32_t code = TDB_CODE_SUCCESS; SMqTopicObj *pTopic = NULL; @@ -596,6 +616,9 @@ END: } static int32_t mndDropTopic(SMnode *pMnode, STrans *pTrans, SRpcMsg *pReq, SMqTopicObj *pTopic) { + if (pMnode == NULL || pTrans == NULL || pReq == NULL || pTopic == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; SSdbRaw *pCommitRaw = NULL; MND_TMQ_RETURN_CHECK(mndUserRemoveTopic(pMnode, pTrans, pTopic->name)); @@ -614,6 +637,9 @@ END: } bool checkTopic(SArray *topics, char *topicName){ + if (topics == NULL || topicName == NULL) { + return false; + } int32_t sz = taosArrayGetSize(topics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(topics, i); @@ -625,6 +651,9 @@ bool checkTopic(SArray *topics, char *topicName){ } static int32_t mndCheckConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){ + if (pMnode == NULL || pTrans == NULL || topicName == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -653,6 +682,9 @@ END: } static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic){ + if (pMnode == NULL || pTrans == NULL || pTopic == NULL) { + return TSDB_CODE_INVALID_MSG; + } // broadcast to all vnode void *pIter = NULL; SVgObj *pVgroup = NULL; @@ -693,6 +725,9 @@ END: } static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { + if (pReq == NULL) { + return TSDB_CODE_INVALID_MSG; + } SMnode *pMnode = pReq->info.node; SMDropTopicReq dropReq = {0}; int32_t code = 0; @@ -756,6 +791,9 @@ END: } int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { + if (pMnode == NULL || dbName == NULL || pNumOfTopics == NULL) { + return TSDB_CODE_INVALID_MSG; + } *pNumOfTopics = 0; SSdb *pSdb = pMnode->pSdb; @@ -786,6 +824,9 @@ int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { } static void schemaToJson(SSchema *schema, int32_t nCols, char *schemaJson){ + if (schema == NULL || schemaJson == NULL) { + return; + } char* string = NULL; int32_t code = 0; cJSON* columns = cJSON_CreateArray(); @@ -838,6 +879,9 @@ END: } static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { + if (pReq == NULL || pShow == NULL || pBlock == NULL) { + return TSDB_CODE_INVALID_MSG; + } SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -945,11 +989,15 @@ END: } static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter) { + if (pMnode == NULL) return; SSdb *pSdb = pMnode->pSdb; sdbCancelFetchByType(pSdb, pIter, SDB_TOPIC); } bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb) { + if (pMnode == NULL || pDb == NULL) { + return false; + } SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SMqTopicObj *pTopic = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index e1518d3752..5b2a5fa8aa 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1805,12 +1805,21 @@ _OVER: TAOS_RETURN(code); } -static int32_t mndCheckPasswordFmt(const char *pwd) { - int32_t len = strlen(pwd); - if (len < TSDB_PASSWORD_MIN_LEN || len > TSDB_PASSWORD_MAX_LEN) { +static int32_t mndCheckPasswordMinLen(const char *pwd, int32_t len) { + if (len < TSDB_PASSWORD_MIN_LEN) { return -1; } + return 0; +} +static int32_t mndCheckPasswordMaxLen(const char *pwd, int32_t len) { + if (len > TSDB_PASSWORD_MAX_LEN) { + return -1; + } + return 0; +} + +static int32_t mndCheckPasswordFmt(const char *pwd, int32_t len) { if (strcmp(pwd, "taosdata") == 0) { return 0; } @@ -1875,14 +1884,17 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (mndCheckPasswordFmt(createReq.pass) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); - } - + int32_t len = strlen(createReq.pass); if (createReq.isImport != 1) { - if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN) { + if (mndCheckPasswordMinLen(createReq.pass, len) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY, &lino, _OVER); + } + if (mndCheckPasswordMaxLen(createReq.pass, len) != 0) { TAOS_CHECK_GOTO(TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG, &lino, _OVER); } + if (mndCheckPasswordFmt(createReq.pass, len) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); + } } code = mndAcquireUser(pMnode, createReq.user, &pUser); @@ -2364,8 +2376,17 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_USER_FORMAT, &lino, _OVER); } - if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && mndCheckPasswordFmt(alterReq.pass) != 0) { - TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); + if (TSDB_ALTER_USER_PASSWD == alterReq.alterType) { + int32_t len = strlen(alterReq.pass); + if (mndCheckPasswordMinLen(alterReq.pass, len) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY, &lino, _OVER); + } + if (mndCheckPasswordMaxLen(alterReq.pass, len) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG, &lino, _OVER); + } + if (mndCheckPasswordFmt(alterReq.pass, len) != 0) { + TAOS_CHECK_GOTO(TSDB_CODE_MND_INVALID_PASS_FORMAT, &lino, _OVER); + } } TAOS_CHECK_GOTO(mndAcquireUser(pMnode, alterReq.user, &pUser), &lino, _OVER); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f4a678e629..b33bdb0976 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -51,7 +51,7 @@ extern const SVnodeCfg vnodeCfgDefault; typedef void (*StopDnodeFp)(); -int32_t vnodeInit(int32_t nthreads, StopDnodeFp stopDnodeFp); +int32_t vnodeInit(StopDnodeFp stopDnodeFp); void vnodeCleanup(); int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs *pTfs); bool vnodeShouldRemoveWal(SVnode *pVnode); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 3c40100f9d..28a0d11757 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -127,11 +127,11 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); int32_t tqMetaOpen(STQ* pTq); void tqMetaClose(STQ* pTq); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); -int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen); -int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen); +int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, uint32_t kLen, const void* value, uint32_t vLen); +int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, uint32_t kLen); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); -int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen); -int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen); +int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, uint32_t vLen); +int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, uint32_t vLen); int32_t tqMetaSaveOffset(STQ* pTq, STqOffset* pOffset); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle); int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 29248e360a..47890e9b4b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -1083,9 +1083,6 @@ void tsdbRemoveFile(const char *path); } \ } while (0) -int32_t tsdbInit(); -void tsdbCleanUp(); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index ff622d2dab..b1a5ca4709 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -55,16 +55,32 @@ typedef enum { EVA_PRIORITY_LOW, } EVAPriority; -int32_t vnodeAsyncOpen(int32_t numOfThreads); +typedef enum { + EVA_TASK_COMMIT = 1, + EVA_TASK_MERGE, + EVA_TASK_COMPACT, + EVA_TASK_RETENTION, +} EVATaskT; + +#define COMMIT_TASK_ASYNC 1 +#define MERGE_TASK_ASYNC 2 +#define COMPACT_TASK_ASYNC 3 +#define RETENTION_TASK_ASYNC 4 + +int32_t vnodeAsyncOpen(); void vnodeAsyncClose(); int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID); int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning); -int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), - void* arg, SVATaskID* taskID); +int32_t vnodeAsync(int64_t async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg, + SVATaskID* taskID); +int32_t vnodeAsyncC(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), + void* arg, SVATaskID* taskID); void vnodeAWait(SVATaskID* taskID); int32_t vnodeACancel(SVATaskID* taskID); int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers); +const char* vnodeGetATaskName(EVATaskT task); + // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; struct SVBufPoolNode { diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ef4b233f94..4f1ecf81dd 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -479,8 +479,7 @@ struct SVnode { SVBufPool* onRecycle; // commit variables - SVAChannelID commitChannel; - SVATaskID commitTask; + SVATaskID commitTask; SMeta* pMeta; SSma* pSma; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bb50499ea8..03037e529b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -25,11 +25,12 @@ // 2: wait to be inited or cleanup static int32_t tqInitialize(STQ* pTq); -static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_STATUS_EXEC == pHandle->status; } -static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; } -static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; } +static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return pHandle != NULL ? TMQ_HANDLE_STATUS_EXEC == pHandle->status : true; } +static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { if (pHandle != NULL) pHandle->status = TMQ_HANDLE_STATUS_EXEC; } +static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { if (pHandle != NULL) pHandle->status = TMQ_HANDLE_STATUS_IDLE; } void tqDestroyTqHandle(void* data) { + if (data == NULL) return; STqHandle* pData = (STqHandle*)data; qDestroyTask(pData->execHandle.task); @@ -59,11 +60,17 @@ void tqDestroyTqHandle(void* data) { } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { + if (pLeft == NULL || pRight == NULL) { + return false; + } return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && pLeft->val.version == pRight->val.version; } int32_t tqOpen(const char* path, SVnode* pVnode) { + if (path == NULL || pVnode == NULL) { + return TSDB_CODE_INVALID_PARA; + } STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { return terrno; @@ -104,6 +111,9 @@ int32_t tqOpen(const char* path, SVnode* pVnode) { } int32_t tqInitialize(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); int32_t code = streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback, &pTq->pStreamMeta); @@ -157,6 +167,9 @@ void tqNotifyClose(STQ* pTq) { } void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { + if (pHandle == NULL) { + return; + } int32_t code = 0; SMqPollReq req = {0}; code = tDeserializeSMqPollReq(pHandle->msg->pCont, pHandle->msg->contLen, &req); @@ -186,6 +199,9 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, int32_t vgId) { + if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) { + return TSDB_CODE_INVALID_PARA; + } int64_t sver = 0, ever = 0; walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); @@ -201,6 +217,9 @@ int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* } int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqVgOffset vgOffset = {0}; int32_t vgId = TD_VID(pTq->pVnode); @@ -243,7 +262,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t } if (tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), msg, - msgLen - sizeof(vgOffset.consumerId)) < 0) { + msgLen >= sizeof(vgOffset.consumerId) ? msgLen - sizeof(vgOffset.consumerId) : 0) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -255,6 +274,9 @@ end: } int32_t tqProcessSeekReq(STQ* pTq, SRpcMsg* pMsg) { + if (pTq == NULL || pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqSeekReq req = {0}; int32_t vgId = TD_VID(pTq->pVnode); SRpcMsg rsp = {.info = pMsg->info}; @@ -297,6 +319,9 @@ end: } int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* pIter = NULL; while (1) { @@ -327,6 +352,9 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { } int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); taosWLockLatch(&pTq->lock); if (taosHashGetSize(pTq->pPushMgr) > 0) { @@ -362,6 +390,9 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { + if (pTq == NULL || pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqPollReq req = {0}; int code = tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req); if (code < 0) { @@ -439,6 +470,9 @@ END: } int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { + if (pTq == NULL || pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* data = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); @@ -485,6 +519,9 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { + if (pTq == NULL || pMsg == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SMqPollReq req = {0}; if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { @@ -570,6 +607,9 @@ END: } int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + if (pTq == NULL || msg == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; int32_t vgId = TD_VID(pTq->pVnode); @@ -616,8 +656,11 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + if (pTq == NULL || msg == NULL) { + return TSDB_CODE_INVALID_PARA; + } STqCheckInfo info = {0}; - int32_t code = tqMetaDecodeCheckInfo(&info, msg, msgLen); + int32_t code = tqMetaDecodeCheckInfo(&info, msg, msgLen >= 0 ? msgLen : 0); if (code != 0) { return code; } @@ -628,10 +671,13 @@ int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t return code; } - return tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen); + return tqMetaSaveInfo(pTq, pTq->pCheckStore, info.topic, strlen(info.topic), msg, msgLen >= 0 ? msgLen : 0); } int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + if (pTq == NULL || msg == NULL) { + return TSDB_CODE_INVALID_PARA; + } if (taosHashRemove(pTq->pCheckInfo, msg, strlen(msg)) < 0) { return TSDB_CODE_TSC_INTERNAL_ERROR; } @@ -639,6 +685,9 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t } int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + if (pTq == NULL || msg == NULL) { + return TSDB_CODE_INVALID_PARA; + } int ret = 0; SMqRebVgReq req = {0}; SDecoder dc = {0}; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 89350e761f..580828b089 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -16,6 +16,9 @@ #include "tq.h" int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle) { + if (pEncoder == NULL || pHandle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t lino; @@ -54,6 +57,9 @@ _exit: } int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) { + if (pDecoder == NULL || pHandle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t lino; @@ -91,7 +97,10 @@ _exit: return code; } -int32_t tqMetaDecodeCheckInfo(STqCheckInfo* info, void* pVal, int32_t vLen) { +int32_t tqMetaDecodeCheckInfo(STqCheckInfo* info, void* pVal, uint32_t vLen) { + if (info == NULL || pVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } SDecoder decoder = {0}; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); int32_t code = tDecodeSTqCheckInfo(&decoder, info); @@ -104,7 +113,10 @@ int32_t tqMetaDecodeCheckInfo(STqCheckInfo* info, void* pVal, int32_t vLen) { return code; } -int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, int32_t vLen) { +int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, uint32_t vLen) { + if (info == NULL || pVal == NULL) { + return TSDB_CODE_INVALID_PARA; + } SDecoder decoder = {0}; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); int32_t code = tDecodeSTqOffset(&decoder, info); @@ -118,9 +130,12 @@ int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, int32_t vLen) { } int32_t tqMetaSaveOffset(STQ* pTq, STqOffset* pOffset) { + if (pTq == NULL || pOffset == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* buf = NULL; int32_t code = TDB_CODE_SUCCESS; - int32_t vlen; + uint32_t vlen; SEncoder encoder = {0}; tEncodeSize(tEncodeSTqOffset, pOffset, vlen, code); if (code < 0) { @@ -147,7 +162,10 @@ END: return code; } -int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) { +int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, uint32_t kLen, const void* value, uint32_t vLen) { + if (pTq == NULL || ttb == NULL || key == NULL || value == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; @@ -164,7 +182,10 @@ END: return code; } -int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen) { +int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, uint32_t kLen) { + if (pTq == NULL || ttb == NULL || key == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; @@ -182,6 +203,9 @@ END: } int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset) { + if (pTq == NULL || subkey == NULL || pOffset == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* data = taosHashGet(pTq->pOffset, subkey, strlen(subkey)); if (data == NULL) { int vLen = 0; @@ -191,7 +215,7 @@ int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset) { } STqOffset offset = {0}; - if (tqMetaDecodeOffsetInfo(&offset, data, vLen) != TDB_CODE_SUCCESS) { + if (tqMetaDecodeOffsetInfo(&offset, data, vLen >= 0 ? vLen : 0) != TDB_CODE_SUCCESS) { tdbFree(data); return TSDB_CODE_OUT_OF_MEMORY; } @@ -214,8 +238,11 @@ int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset) { } int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { + if (pTq == NULL || key == NULL || pHandle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; - int32_t vlen; + uint32_t vlen; void* buf = NULL; SEncoder encoder = {0}; tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); @@ -238,7 +265,7 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { goto END; } - TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, (int)strlen(key), buf, vlen)); + TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pExecStore, key, strlen(key), buf, vlen)); END: tEncoderClear(&encoder); @@ -247,6 +274,9 @@ END: } static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) { + if (pTq == NULL || handle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; SVnode* pVnode = pTq->pVnode; @@ -318,7 +348,10 @@ END: return code; } -static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, int vLen, STqHandle* handle) { +static int32_t tqMetaRestoreHandle(STQ* pTq, void* pVal, uint32_t vLen, STqHandle* handle) { + if (pTq == NULL || pVal == NULL || handle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); SDecoder decoder = {0}; int32_t code = TDB_CODE_SUCCESS; @@ -335,6 +368,9 @@ END: } int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) { + if (pTq == NULL || req == NULL || handle == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); (void)memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -375,6 +411,9 @@ int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle) { } static int32_t tqMetaTransformInfo(TDB* pMetaDB, TTB* pOld, TTB* pNew) { + if (pMetaDB == NULL || pOld == NULL || pNew == NULL) { + return TSDB_CODE_INVALID_PARA; + } TBC* pCur = NULL; void* pKey = NULL; int kLen = 0; @@ -404,6 +443,9 @@ END: } int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { + if (pTq == NULL || key == NULL || pHandle == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* data = taosHashGet(pTq->pHandle, key, strlen(key)); if (data == NULL) { int vLen = 0; @@ -412,7 +454,7 @@ int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { return TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; } STqHandle handle = {0}; - if (tqMetaRestoreHandle(pTq, data, vLen, &handle) != 0) { + if (tqMetaRestoreHandle(pTq, data, vLen >= 0 ? vLen : 0, &handle) != 0) { tdbFree(data); tqDestroyTqHandle(&handle); return TSDB_CODE_OUT_OF_MEMORY; @@ -429,6 +471,9 @@ int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle) { } int32_t tqMetaOpenTdb(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; TQ_ERR_GO_TO_END(tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL)); TQ_ERR_GO_TO_END(tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0)); @@ -440,6 +485,9 @@ END: } static int32_t replaceTqPath(char** path) { + if (path == NULL || *path == NULL) { + return TSDB_CODE_INVALID_PARA; + } char* tpath = NULL; int32_t code = tqBuildFName(&tpath, *path, TQ_SUBSCRIBE_NAME); if (code != 0) { @@ -451,6 +499,9 @@ static int32_t replaceTqPath(char** path) { } static int32_t tqMetaRestoreCheckInfo(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } TBC* pCur = NULL; void* pKey = NULL; int kLen = 0; @@ -463,7 +514,7 @@ static int32_t tqMetaRestoreCheckInfo(STQ* pTq) { TQ_ERR_GO_TO_END(tdbTbcMoveToFirst(pCur)); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { - TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen)); + TQ_ERR_GO_TO_END(tqMetaDecodeCheckInfo(&info, pVal, vLen >= 0 ? vLen : 0)); TQ_ERR_GO_TO_END(taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo))); } info.colIdList = NULL; @@ -477,6 +528,9 @@ END: } int32_t tqMetaOpen(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } char* maindb = NULL; char* offsetNew = NULL; int32_t code = TDB_CODE_SUCCESS; @@ -504,6 +558,9 @@ END: } int32_t tqMetaTransform(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = TDB_CODE_SUCCESS; TDB* pMetaDB = NULL; TTB* pExecStore = NULL; @@ -543,6 +600,9 @@ END: } void tqMetaClose(STQ* pTq) { + if (pTq == NULL) { + return; + } int32_t ret = 0; if (pTq->pExecStore) { tdbTbClose(pTq->pExecStore); diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index f392269b9f..c42959971b 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -17,6 +17,9 @@ #include "tq.h" int32_t tqBuildFName(char** data, const char* path, char* name) { + if (data == NULL || path == NULL || name == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t len = strlen(path) + strlen(name) + 2; char* fname = taosMemoryCalloc(1, len); if(fname == NULL) { @@ -33,6 +36,9 @@ int32_t tqBuildFName(char** data, const char* path, char* name) { } int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { + if (pTq == NULL || name == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = TDB_CODE_SUCCESS; void* pMemBuf = NULL; @@ -54,6 +60,10 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { } total += INT_BYTES; size = htonl(size); + if (size <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto END; + } pMemBuf = taosMemoryCalloc(1, size); if (pMemBuf == NULL) { code = terrno; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 386b61cce3..2b2667773a 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -17,6 +17,9 @@ #include "vnd.h" int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { + if (pTq == NULL) { + return TSDB_CODE_INVALID_MSG; + } if (taosHashGetSize(pTq->pPushMgr) <= 0) { return 0; } @@ -64,6 +67,9 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) { } int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { + if (pTq == NULL || handle == NULL || pMsg == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t vgId = TD_VID(pTq->pVnode); STqHandle* pHandle = (STqHandle*)handle; @@ -101,6 +107,9 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { } void tqUnregisterPushHandle(STQ* pTq, void *handle) { + if (pTq == NULL || handle == NULL) { + return; + } STqHandle *pHandle = (STqHandle*)handle; int32_t vgId = TD_VID(pTq->pVnode); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index d924e97ae3..c0c4c4a5a3 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -17,6 +17,9 @@ #include "tq.h" bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { + if (pHandle == NULL || pHead == NULL) { + return false; + } if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__TABLE) { return true; } @@ -198,6 +201,9 @@ end: } int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId) { + if (pTq == NULL || pHandle == NULL || fetchOffset == NULL) { + return -1; + } int32_t code = -1; int32_t vgId = TD_VID(pTq->pVnode); int64_t id = pHandle->pWalReader->readerId; @@ -259,9 +265,17 @@ END: return code; } -bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; } +bool tqGetTablePrimaryKey(STqReader* pReader) { + if (pReader == NULL) { + return false; + } + return pReader->hasPrimaryKey; +} void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { + if (pReader == NULL) { + return; + } bool ret = false; SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { @@ -272,6 +286,9 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { } STqReader* tqReaderOpen(SVnode* pVnode) { + if (pVnode == NULL) { + return NULL; + } STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; @@ -323,6 +340,9 @@ void tqReaderClose(STqReader* pReader) { } int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { + if (pReader == NULL) { + return TSDB_CODE_INVALID_PARA; + } if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { return -1; } @@ -406,6 +426,9 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con } bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { + if (pReader == NULL) { + return false; + } SWalReader* pWalReader = pReader->pWalReader; int64_t st = taosGetTimestampMs(); @@ -462,6 +485,9 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { } int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { +if (pReader == NULL) { + return TSDB_CODE_INVALID_PARA; + } pReader->msg.msgStr = msgStr; pReader->msg.msgLen = msgLen; pReader->msg.ver = ver; @@ -481,14 +507,29 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } -SWalReader* tqGetWalReader(STqReader* pReader) { return pReader->pWalReader; } +SWalReader* tqGetWalReader(STqReader* pReader) { + if (pReader == NULL) { + return NULL; + } + return pReader->pWalReader; +} -SSDataBlock* tqGetResultBlock(STqReader* pReader) { return pReader->pResBlock; } +SSDataBlock* tqGetResultBlock(STqReader* pReader) { + if (pReader == NULL) { + return NULL; + } + return pReader->pResBlock; +} -int64_t tqGetResultBlockTime(STqReader* pReader) { return pReader->lastTs; } +int64_t tqGetResultBlockTime(STqReader* pReader) { + if (pReader == NULL) { + return 0; + } + return pReader->lastTs; +} bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { - if (pReader->msg.msgStr == NULL) { + if (pReader == NULL || pReader->msg.msgStr == NULL) { return false; } @@ -525,7 +566,7 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { } bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { - if (pReader->msg.msgStr == NULL) return false; + if (pReader == NULL || pReader->msg.msgStr == NULL) return false; int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { @@ -548,6 +589,9 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { } int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { + if (pDst == NULL || pBlock == NULL || pSrc == NULL || mask == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t cnt = 0; @@ -577,6 +621,9 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap } static int32_t buildResSDataBlock(STqReader* pReader, SSchemaWrapper* pSchema, const SArray* pColIdList) { + if (pReader == NULL || pSchema == NULL || pColIdList == NULL) { + return TSDB_CODE_INVALID_PARA; + } SSDataBlock* pBlock = pReader->pResBlock; if (blockDataGetNumOfCols(pBlock) > 0) { blockDataDestroy(pBlock); @@ -659,6 +706,9 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol } int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* id) { + if (pReader == NULL || pRes == NULL) { + return TSDB_CODE_INVALID_PARA; + } tqTrace("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); int32_t code = 0; int32_t line = 0; @@ -825,6 +875,10 @@ END: static int32_t processBuildNew(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas, SSchemaWrapper* pSchemaWrapper, char* assigned, int32_t numOfRows, int32_t curRow, int32_t* lastRow) { + if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL || pSchemaWrapper == NULL || + assigned == NULL || lastRow == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; SSchemaWrapper* pSW = NULL; SSDataBlock* block = NULL; @@ -860,6 +914,9 @@ END: return code; } static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { + if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; int32_t curRow = 0; int32_t lastRow = 0; @@ -919,6 +976,9 @@ END: } int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArray* blocks, SArray* schemas) { + if (pReader == NULL || pSubmitTbData == NULL || blocks == NULL || schemas == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; STSchema* pTSchema = NULL; @@ -976,6 +1036,9 @@ END: } int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) { + if (pReader == NULL || blocks == NULL || schemas == NULL) { + return TSDB_CODE_INVALID_PARA; + } tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pSubmitTbData == NULL) { @@ -1007,9 +1070,17 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas } } -void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } +void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { + if (pReader == NULL){ + return; + } + pReader->pColIdList = pColIdList; +} void tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char* id) { + if (pReader == NULL || tbUidList == NULL) { + return; + } if (pReader->tbIdHash) { taosHashClear(pReader->tbIdHash); } else { @@ -1032,6 +1103,9 @@ void tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const cha } void tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { + if (pReader == NULL || pTableUidList == NULL) { + return; + } if (pReader->tbIdHash == NULL) { pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pReader->tbIdHash == NULL) { @@ -1051,12 +1125,23 @@ void tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { } bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) { - return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)); + if (pReader == NULL) { + return false; + } + return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)) != NULL; } -bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgStr == NULL; } +bool tqCurrentBlockConsumed(const STqReader* pReader) { + if (pReader == NULL) { + return false; + } + return pReader->msg.msgStr == NULL; +} void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { + if (pReader == NULL || tbUidList == NULL) { + return; + } for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); if (pKey && taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)) != 0) { @@ -1066,6 +1151,9 @@ void tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { } int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { + if (pTq == NULL || tbUidList == NULL) { + return TSDB_CODE_INVALID_PARA; + } void* pIter = NULL; int32_t vgId = TD_VID(pTq->pVnode); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 39eaac39b3..3419cd0020 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -16,6 +16,9 @@ #include "tq.h" int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { + if (pBlock == NULL || pRsp == NULL) { + return TSDB_CODE_INVALID_PARA; + } size_t dataEncodeBufSize = blockGetEncodeSize(pBlock); int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + dataEncodeBufSize; void* buf = taosMemoryCalloc(1, dataStrLen); @@ -47,18 +50,10 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t return TSDB_CODE_SUCCESS; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRsp) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); - if (pSW == NULL) { - return terrno; - } - if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL) { - return terrno; - } - return 0; -} - static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { + if (pRsp == NULL || pTq == NULL) { + return TSDB_CODE_INVALID_PARA; + } SMetaReader mr = {0}; metaReaderDoInit(&mr, pTq->pVnode->pMeta, META_READER_LOCK); @@ -84,6 +79,9 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i } int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res) { + if (task == NULL || pHandle == NULL || res == NULL) { + return TSDB_CODE_INVALID_PARA; + } uint64_t ts = 0; qStreamSetOpen(task); @@ -99,6 +97,9 @@ int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, S } int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) { + if (pTq == NULL || pHandle == NULL || pRsp == NULL || pOffset == NULL || pRequest == NULL){ + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); int32_t code = 0; int32_t line = 0; @@ -189,6 +190,9 @@ END: } int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { + if (pTq == NULL || pHandle == NULL || pRsp == NULL || pBatchMetaRsp == NULL || pOffset == NULL) { + return TSDB_CODE_INVALID_PARA; + } const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; int code = qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); @@ -280,6 +284,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat } static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ + if (pRsp == NULL || pCreateTbReq == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t code = 0; void* createReq = NULL; if (pRsp->createTableNum == 0) { @@ -329,6 +336,9 @@ END: } static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ + if (pTq == NULL || pHandle == NULL || pRsp == NULL || totalRows == NULL) { + return; + } int32_t code = 0; STqExecHandle* pExec = &pHandle->execHandle; STqReader* pReader = pExec->pTqReader; @@ -407,6 +417,9 @@ END: int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { + if (pTq == NULL || pHandle == NULL || pRsp == NULL || totalRows == NULL) { + return TSDB_CODE_INVALID_PARA; + } STqExecHandle* pExec = &pHandle->execHandle; int32_t code = 0; STqReader* pReader = pExec->pTqReader; diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index cfa97def74..219ea4b6b4 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -27,6 +27,9 @@ struct STqSnapReader { }; int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) { + if (pTq == NULL || ppReader == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STqSnapReader* pReader = NULL; @@ -77,12 +80,18 @@ _err: } void tqSnapReaderClose(STqSnapReader** ppReader) { + if (ppReader == NULL || *ppReader == NULL) { + return; + } tdbTbcClose((*ppReader)->pCur); taosMemoryFree(*ppReader); *ppReader = NULL; } int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { + if (pReader == NULL || ppData == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; void* pKey = NULL; void* pVal = NULL; @@ -126,6 +135,9 @@ struct STqSnapWriter { }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { + if (pTq == NULL || ppWriter == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STqSnapWriter* pWriter = NULL; @@ -156,6 +168,9 @@ _err: } int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { + if (ppWriter == NULL || *ppWriter == NULL) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STqSnapWriter* pWriter = *ppWriter; STQ* pTq = pWriter->pTq; @@ -180,6 +195,9 @@ _err: } int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STQ* pTq = pWriter->pTq; SDecoder decoder = {0}; @@ -190,7 +208,7 @@ int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData code = tDecodeSTqHandle(pDecoder, &handle); if (code) goto end; taosWLockLatch(&pTq->lock); - code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, (int)strlen(handle.subKey), pData + sizeof(SSnapDataHdr), + code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, strlen(handle.subKey), pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); taosWUnLockLatch(&pTq->lock); @@ -202,6 +220,9 @@ end: } int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STQ* pTq = pWriter->pTq; STqCheckInfo info = {0}; @@ -223,6 +244,9 @@ _err: } int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { + if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) { + return TSDB_CODE_INVALID_MSG; + } int32_t code = 0; STQ* pTq = pWriter->pTq; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index a92049e5f3..f6a8563c70 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -21,6 +21,9 @@ static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, c const SMqBatchMetaRsp* pRsp, int32_t vgId); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { + if (pRsp == NULL) { + return TSDB_CODE_INVALID_PARA; + } pRsp->blockData = taosArrayInit(0, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); @@ -41,6 +44,9 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { } static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { + if (pRsp == NULL) { + return TSDB_CODE_INVALID_PARA; + } tOffsetCopy(&pRsp->reqOffset, &pOffset); tOffsetCopy(&pRsp->rspOffset, &pOffset); @@ -81,6 +87,9 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { + if (pOffsetVal == NULL || pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL || pBlockReturned == NULL) { + return TSDB_CODE_INVALID_PARA; + } uint64_t consumerId = pRequest->consumerId; STqOffset* pOffset = NULL; int32_t code = tqMetaGetOffset(pTq, pRequest->subKey, &pOffset); @@ -142,6 +151,9 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* pOffset) { + if (pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL || pOffset == NULL) { + return TSDB_CODE_INVALID_PARA; + } uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); terrno = 0; @@ -212,6 +224,9 @@ static void tDeleteCommon(void* parm) {} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { + if (pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL || offset == NULL) { + return TSDB_CODE_INVALID_PARA; + } int32_t vgId = TD_VID(pTq->pVnode); SMqDataRsp taosxRsp = {0}; SMqBatchMetaRsp btMetaRsp = {0}; @@ -410,6 +425,9 @@ END: } int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { + if (pTq == NULL || pHandle == NULL || pRequest == NULL || pMsg == NULL) { + return TSDB_CODE_TMQ_INVALID_MSG; + } int32_t code = 0; STqOffsetVal reqOffset = {0}; tOffsetCopy(&reqOffset, &pRequest->reqOffset); @@ -445,6 +463,9 @@ END: static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int64_t consumerId, int64_t sver, int64_t ever) { + if (pMsgHead == NULL) { + return; + } pMsgHead->consumerId = consumerId; pMsgHead->epoch = epoch; pMsgHead->mqMsgType = type; @@ -454,6 +475,9 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, int32_t vgId) { + if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) { + return TSDB_CODE_TMQ_INVALID_MSG; + } int32_t len = 0; int32_t code = 0; tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code); @@ -491,6 +515,9 @@ int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SM int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId) { + if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) { + return TSDB_CODE_TMQ_INVALID_MSG; + } int32_t len = 0; int32_t code = 0; tEncodeSize(tEncodeMqMetaRsp, pRsp, len, code); @@ -529,6 +556,9 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { + if (pRpcHandleInfo == NULL || pRsp == NULL) { + return TSDB_CODE_TMQ_INVALID_MSG; + } int32_t len = 0; int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8fd0d47969..2047b68101 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -2635,23 +2635,15 @@ _exit: } int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) { - int32_t code = 0, lino = 0; - // fetch schema + int32_t code = 0, lino = 0; STSchema *pTSchema = NULL; int sver = -1; + int numKeys = 0; + SArray *remainCols = NULL; TAOS_CHECK_RETURN(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema)); - // build keys & multi get from rocks - int numCols = pTSchema->numOfCols; - int numKeys = 0; - SArray *remainCols = NULL; - - code = tsdbCacheCommit(pTsdb); - if (code != TSDB_CODE_SUCCESS) { - tsdbTrace("vgId:%d, %s commit failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, - tstrerror(code)); - } + int numCols = pTSchema->numOfCols; (void)taosThreadMutexLock(&pTsdb->lruMutex); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index e3c75760c8..5822463f9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -574,7 +574,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { // begin tasks on file set for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); - tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); + tsdbBeginTaskOnFileSet(tsdb, info->fid, EVA_TASK_COMMIT, &fset); if (fset) { code = tsdbTFileSetInitCopy(tsdb, fset, &info->fset); if (code) { @@ -712,7 +712,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); if (info->fset) { - tsdbFinishTaskOnFileSet(tsdb, info->fid); + tsdbFinishTaskOnFileSet(tsdb, info->fid, EVA_TASK_COMMIT); } } @@ -743,7 +743,7 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) { for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); if (info->fset) { - tsdbFinishTaskOnFileSet(pTsdb, info->fid); + tsdbFinishTaskOnFileSet(pTsdb, info->fid, EVA_TASK_COMMIT); } } (void)taosThreadMutexUnlock(&pTsdb->mutex); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 9a7cdca8f7..82dd49b0e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -770,8 +770,8 @@ extern void tsdbStopAllCompTask(STsdb *tsdb); int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { STFileSystem *fs = pTsdb->pFS; - SArray *channelArray = taosArrayInit(0, sizeof(SVAChannelID)); - if (channelArray == NULL) { + SArray *asyncTasks = taosArrayInit(0, sizeof(SVATaskID)); + if (asyncTasks == NULL) { return terrno; } @@ -783,30 +783,31 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { // collect channel STFileSet *fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - if (fset->channelOpened) { - if (taosArrayPush(channelArray, &fset->channel) == NULL) { - taosArrayDestroy(channelArray); - (void)taosThreadMutexUnlock(&pTsdb->mutex); - return terrno; - } - fset->channel = (SVAChannelID){0}; - fset->mergeScheduled = false; - tsdbFSSetBlockCommit(fset, false); - fset->channelOpened = false; + if (taosArrayPush(asyncTasks, &fset->mergeTask) == NULL // + || taosArrayPush(asyncTasks, &fset->compactTask) == NULL // + || taosArrayPush(asyncTasks, &fset->retentionTask) == NULL) { + taosArrayDestroy(asyncTasks); + (void)taosThreadMutexUnlock(&pTsdb->mutex); + return terrno; } + fset->mergeScheduled = false; + tsdbFSSetBlockCommit(fset, false); } (void)taosThreadMutexUnlock(&pTsdb->mutex); // destroy all channels - for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) { - SVAChannelID *channel = taosArrayGet(channelArray, i); - int32_t code = vnodeAChannelDestroy(channel, true); - if (code) { - tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + for (int32_t k = 0; k < 2; k++) { + for (int32_t i = 0; i < taosArrayGetSize(asyncTasks); i++) { + SVATaskID *task = taosArrayGet(asyncTasks, i); + if (k == 0) { + (void)vnodeACancel(task); + } else { + vnodeAWait(task); + } } } - taosArrayDestroy(channelArray); + taosArrayDestroy(asyncTasks); #ifdef TD_ENTERPRISE tsdbStopAllCompTask(pTsdb); @@ -934,9 +935,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { // bool skipMerge = false; int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); if (numFile >= sttTrigger && (!fset->mergeScheduled)) { - code = tsdbTFileSetOpenChannel(fset); - TSDB_CHECK_CODE(code, lino, _exit); - SMergeArg *arg = taosMemoryMalloc(sizeof(*arg)); if (arg == NULL) { code = terrno; @@ -946,7 +944,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { arg->tsdb = fs->tsdb; arg->fid = fset->fid; - code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosAutoMemoryFree, arg, NULL); + code = vnodeAsync(MERGE_TASK_ASYNC, EVA_PRIORITY_HIGH, tsdbMerge, taosAutoMemoryFree, arg, &fset->mergeTask); TSDB_CHECK_CODE(code, lino, _exit); fset->mergeScheduled = true; } @@ -1202,42 +1200,61 @@ _out: void tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { tsdbTFileSetRangeArrayDestroy(fsrArr); } -void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) { +void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, EVATaskT task, STFileSet **fset) { + // Here, sttTrigger is protected by tsdb->mutex, so it is safe to read it without lock int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; tsdbFSGetFSet(tsdb->pFS, fid, fset); - if (sttTrigger == 1 && (*fset)) { - for (;;) { - if ((*fset)->taskRunning) { - (*fset)->numWaitTask++; - - (void)taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex); - - tsdbFSGetFSet(tsdb->pFS, fid, fset); - - (*fset)->numWaitTask--; - } else { - (*fset)->taskRunning = true; - break; - } - } - tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid); + if (*fset == NULL) { + return; } + + struct STFileSetCond *cond = NULL; + if (sttTrigger == 1 || task == EVA_TASK_COMMIT) { + cond = &(*fset)->conds[0]; + } else { + cond = &(*fset)->conds[1]; + } + + while (1) { + if (cond->running) { + cond->numWait++; + (void)taosThreadCondWait(&cond->cond, &tsdb->mutex); + cond->numWait--; + } else { + cond->running = true; + break; + } + } + + tsdbInfo("vgId:%d begin %s task on file set:%d", TD_VID(tsdb->pVnode), vnodeGetATaskName(task), fid); + return; } -void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { +void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid, EVATaskT task) { + // Here, sttTrigger is protected by tsdb->mutex, so it is safe to read it without lock int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; - if (sttTrigger == 1) { - STFileSet *fset = NULL; - tsdbFSGetFSet(tsdb->pFS, fid, &fset); - if (fset != NULL && fset->taskRunning) { - fset->taskRunning = false; - if (fset->numWaitTask > 0) { - (void)taosThreadCondSignal(&fset->beginTask); - } - tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid); - } + + STFileSet *fset = NULL; + tsdbFSGetFSet(tsdb->pFS, fid, &fset); + if (fset == NULL) { + return; } + + struct STFileSetCond *cond = NULL; + if (sttTrigger == 1 || task == EVA_TASK_COMMIT) { + cond = &fset->conds[0]; + } else { + cond = &fset->conds[1]; + } + + cond->running = false; + if (cond->numWait > 0) { + (void)taosThreadCondSignal(&cond->cond); + } + + tsdbInfo("vgId:%d finish %s task on file set:%d", TD_VID(tsdb->pVnode), vnodeGetATaskName(task), fid); + return; } struct SFileSetReader { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 119015636b..9694edcdd9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -14,6 +14,7 @@ */ #include "tsdbFSet2.h" +#include "vnd.h" #ifndef _TSDB_FILE_SYSTEM_H #define _TSDB_FILE_SYSTEM_H @@ -61,8 +62,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs); // other void tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); void tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); -void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset); -void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid); +void tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, EVATaskT task, STFileSet **fset); +void tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid, EVATaskT task); // utils int32_t save_fs(const TFileSetArray *arr, const char *fname); void current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index a0ae58ac96..68914300e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -480,16 +480,18 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { fset[0]->maxVerValid = VERSION_MAX; TARRAY2_INIT(fset[0]->lvlArr); - // background task queue - (void)taosThreadCondInit(&(*fset)->beginTask, NULL); - (*fset)->taskRunning = false; - (*fset)->numWaitTask = 0; - // block commit variables (void)taosThreadCondInit(&fset[0]->canCommit, NULL); (*fset)->numWaitCommit = 0; (*fset)->blockCommit = false; + for (int32_t i = 0; i < sizeof((*fset)->conds) / sizeof((*fset)->conds[0]); ++i) { + struct STFileSetCond *cond = &(*fset)->conds[i]; + cond->running = false; + cond->numWait = 0; + (void)taosThreadCondInit(&cond->cond, NULL); + } + return 0; } @@ -648,8 +650,10 @@ void tsdbTFileSetClear(STFileSet **fset) { TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear); - (void)taosThreadCondDestroy(&(*fset)->beginTask); (void)taosThreadCondDestroy(&(*fset)->canCommit); + for (int32_t i = 0; i < sizeof((*fset)->conds) / sizeof((*fset)->conds[0]); ++i) { + (void)taosThreadCondDestroy(&(*fset)->conds[i].cond); + } taosMemoryFreeClear(*fset); } } @@ -703,14 +707,3 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { } return TARRAY2_SIZE(fset->lvlArr) == 0; } - -int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { - int32_t code; - if (!fset->channelOpened) { - if ((code = vnodeAChannelInit(2, &fset->channel))) { - return code; - } - fset->channelOpened = true; - } - return 0; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 24ae59e300..83ef32e5e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -68,8 +68,6 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset); // stt int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl); void tsdbSttLvlClear(SSttLvl **lvl); -// open channel -int32_t tsdbTFileSetOpenChannel(STFileSet *fset); struct STFileOp { tsdb_fop_t optype; @@ -83,26 +81,30 @@ struct SSttLvl { TFileObjArray fobjArr[1]; }; +struct STFileSetCond { + bool running; + int32_t numWait; + TdThreadCond cond; +}; + struct STFileSet { int32_t fid; int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array - // background task - bool channelOpened; - SVAChannelID channel; - bool mergeScheduled; - - // sttTrigger = 1 - TdThreadCond beginTask; - bool taskRunning; - int32_t numWaitTask; + bool mergeScheduled; + SVATaskID mergeTask; + SVATaskID compactTask; + SVATaskID retentionTask; // block commit variables TdThreadCond canCommit; int32_t numWaitCommit; bool blockCommit; + + // conditions + struct STFileSetCond conds[2]; }; struct STFileSetRange { diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 61a82d828e..39d8a57692 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -462,21 +462,29 @@ _exit: static int32_t tsdbMergeGetFSet(SMerger *merger) { STFileSet *fset; + int32_t code; + STsdb *tsdb = merger->tsdb; (void)taosThreadMutexLock(&merger->tsdb->mutex); - tsdbFSGetFSet(merger->tsdb->pFS, merger->fid, &fset); - if (fset == NULL) { + + if (tsdb->bgTaskDisabled) { (void)taosThreadMutexUnlock(&merger->tsdb->mutex); return 0; } - fset->mergeScheduled = false; + tsdbBeginTaskOnFileSet(tsdb, merger->fid, EVA_TASK_MERGE, &fset); + if (NULL == fset) { + (void)taosThreadMutexUnlock(&merger->tsdb->mutex); + return 0; + } - int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset); + code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset); if (code) { (void)taosThreadMutexUnlock(&merger->tsdb->mutex); return code; } + + fset->mergeScheduled = false; (void)taosThreadMutexUnlock(&merger->tsdb->mutex); return 0; } @@ -493,10 +501,13 @@ int32_t tsdbMerge(void *arg) { .sttTrigger = tsdb->pVnode->config.sttTrigger, }}; - if (merger->sttTrigger <= 1) return 0; + if (merger->sttTrigger <= 1) { + return 0; + } // copy snapshot - TAOS_CHECK_GOTO(tsdbMergeGetFSet(merger), &lino, _exit); + code = tsdbMergeGetFSet(merger); + TSDB_CHECK_CODE(code, lino, _exit); if (merger->fset == NULL) { return 0; @@ -509,12 +520,19 @@ int32_t tsdbMerge(void *arg) { TSDB_CHECK_CODE(code, lino, _exit); _exit: + if (merger->fset) { + (void)taosThreadMutexLock(&tsdb->mutex); + tsdbFinishTaskOnFileSet(tsdb, mergeArg->fid, EVA_TASK_MERGE); + (void)taosThreadMutexUnlock(&tsdb->mutex); + } + if (code) { tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); tsdbFatal("vgId:%d, failed to merge stt files since %s. code:%d", TD_VID(tsdb->pVnode), terrstr(), code); taosMsleep(100); exit(EXIT_FAILURE); } + tsdbTFileSetClear(&merger->fset); taosMemoryFree(arg); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index b2e4621878..c1f8f45d7e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -18,22 +18,6 @@ extern int32_t tsdbOpenCompMonitor(STsdb *tsdb); extern void tsdbCloseCompMonitor(STsdb *tsdb); -extern int32_t tsdbInitCompact(); -extern void tsdbCleanupCompact(); - -int32_t tsdbInit() { -#ifdef TD_ENTERPRISE - return tsdbInitCompact(); -#endif - return 0; -} - -void tsdbCleanUp() { -#ifdef TD_ENTERPRISE - tsdbCleanupCompact(); -#endif - return; -} void tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 7859ee4c66..fcce36b121 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -325,11 +325,21 @@ static int32_t tsdbRetention(void *arg) { // begin task (void)taosThreadMutexLock(&pTsdb->mutex); - tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset); + + // check if background task is disabled + if (pTsdb->bgTaskDisabled) { + tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(pTsdb->pVnode)); + (void)taosThreadMutexUnlock(&pTsdb->mutex); + return 0; + } + + // set flag and copy + tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION, &fset); if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) { (void)taosThreadMutexUnlock(&pTsdb->mutex); TSDB_CHECK_CODE(code, lino, _exit); } + (void)taosThreadMutexUnlock(&pTsdb->mutex); // do retention @@ -346,7 +356,7 @@ static int32_t tsdbRetention(void *arg) { _exit: if (rtner.fset) { (void)taosThreadMutexLock(&pTsdb->mutex); - tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid); + tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION); (void)taosThreadMutexUnlock(&pTsdb->mutex); } @@ -364,26 +374,29 @@ static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) int32_t code = 0; int32_t lino = 0; + // check if background task is disabled + if (tsdb->bgTaskDisabled) { + tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(tsdb->pVnode)); + return 0; + } + STFileSet *fset; + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } - if (!tsdb->bgTaskDisabled) { - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - TAOS_CHECK_GOTO(tsdbTFileSetOpenChannel(fset), &lino, _exit); + arg->tsdb = tsdb; + arg->now = now; + arg->fid = fset->fid; + arg->s3Migrate = s3Migrate; - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } - - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; - arg->s3Migrate = s3Migrate; - - if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) { - taosMemoryFree(arg); - TSDB_CHECK_CODE(code, lino, _exit); - } + code = vnodeAsync(RETENTION_TASK_ASYNC, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, + &fset->retentionTask); + if (code) { + taosMemoryFree(arg); + TSDB_CHECK_CODE(code, lino, _exit); } } diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 3f40ace6c4..49c1306736 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -118,9 +118,19 @@ struct SVAsync { SVHashTable *taskTable; }; -SVAsync *vnodeAsyncs[3]; +struct { + const char *label; + SVAsync *async; +} GVnodeAsyncs[] = { + [0] = {NULL, NULL}, + [1] = {"vnode-commit", NULL}, + [2] = {"vnode-merge", NULL}, + [3] = {"vnode-compact", NULL}, + [4] = {"vnode-retention", NULL}, +}; + #define MIN_ASYNC_ID 1 -#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1) +#define MAX_ASYNC_ID (sizeof(GVnodeAsyncs) / sizeof(GVnodeAsyncs[0]) - 1) static void vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { int32_t ret; @@ -447,36 +457,47 @@ static void vnodeAsyncLaunchWorker(SVAsync *async) { } } -int32_t vnodeAsyncOpen(int32_t numOfThreads) { +int32_t vnodeAsyncOpen() { int32_t code = 0; int32_t lino = 0; - // vnode-commit - code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit"); - TSDB_CHECK_CODE(code, lino, _exit); + int32_t numOfThreads[] = { + 0, // + tsNumOfCommitThreads, // vnode-commit + tsNumOfCommitThreads, // vnode-merge + tsNumOfCompactThreads, // vnode-compact + tsNumOfRetentionThreads, // vnode-retention + }; - code = vnodeAsyncSetWorkers(1, numOfThreads); - TSDB_CHECK_CODE(code, lino, _exit); + for (int32_t i = 1; i < sizeof(GVnodeAsyncs) / sizeof(GVnodeAsyncs[0]); i++) { + code = vnodeAsyncInit(&GVnodeAsyncs[i].async, GVnodeAsyncs[i].label); + TSDB_CHECK_CODE(code, lino, _exit); - // vnode-merge - code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge"); - TSDB_CHECK_CODE(code, lino, _exit); - - code = vnodeAsyncSetWorkers(2, numOfThreads); - TSDB_CHECK_CODE(code, lino, _exit); + code = vnodeAsyncSetWorkers(i, numOfThreads[i]); + TSDB_CHECK_CODE(code, lino, _exit); + } _exit: return code; } void vnodeAsyncClose() { - int32_t ret; - ret = vnodeAsyncDestroy(&vnodeAsyncs[1]); - ret = vnodeAsyncDestroy(&vnodeAsyncs[2]); + for (int32_t i = 1; i < sizeof(GVnodeAsyncs) / sizeof(GVnodeAsyncs[0]); i++) { + int32_t ret = vnodeAsyncDestroy(&GVnodeAsyncs[i].async); + } } -int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *), - void *arg, SVATaskID *taskID) { +int32_t vnodeAsync(int64_t async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *), void *arg, + SVATaskID *taskID) { + SVAChannelID channelID = { + .async = async, + .id = 0, + }; + return vnodeAsyncC(&channelID, priority, execute, complete, arg, taskID); +} + +int32_t vnodeAsyncC(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *), + void *arg, SVATaskID *taskID) { if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL || channelID->id < 0) { return TSDB_CODE_INVALID_PARA; @@ -484,7 +505,7 @@ int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*exec int32_t ret; int64_t id; - SVAsync *async = vnodeAsyncs[channelID->async]; + SVAsync *async = GVnodeAsyncs[channelID->async].async; // create task object SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask)); @@ -594,7 +615,7 @@ void vnodeAWait(SVATaskID *taskID) { return; } - SVAsync *async = vnodeAsyncs[taskID->async]; + SVAsync *async = GVnodeAsyncs[taskID->async].async; SVATask *task = NULL; SVATask task2 = { .taskId = taskID->id, @@ -623,7 +644,7 @@ int32_t vnodeACancel(SVATaskID *taskID) { } int32_t ret = 0; - SVAsync *async = vnodeAsyncs[taskID->async]; + SVAsync *async = GVnodeAsyncs[taskID->async].async; SVATask *task = NULL; SVATask task2 = { .taskId = taskID->id, @@ -660,7 +681,7 @@ int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) { return TSDB_CODE_INVALID_PARA; } int32_t ret; - SVAsync *async = vnodeAsyncs[asyncID]; + SVAsync *async = GVnodeAsyncs[asyncID].async; (void)taosThreadMutexLock(&async->mutex); async->numWorkers = numWorkers; if (async->numIdleWorkers > 0) { @@ -676,7 +697,7 @@ int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) { return TSDB_CODE_INVALID_PARA; } - SVAsync *async = vnodeAsyncs[asyncID]; + SVAsync *async = GVnodeAsyncs[asyncID].async; // create channel object SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel)); @@ -722,7 +743,7 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { return TSDB_CODE_INVALID_PARA; } - SVAsync *async = vnodeAsyncs[channelID->async]; + SVAsync *async = GVnodeAsyncs[channelID->async].async; SVAChannel *channel = NULL; SVAChannel channel2 = { .channelId = channelID->id, @@ -806,4 +827,19 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { channelID->async = 0; channelID->id = 0; return 0; +} + +const char *vnodeGetATaskName(EVATaskT taskType) { + switch (taskType) { + case EVA_TASK_COMMIT: + return "vnode-commit"; + case EVA_TASK_MERGE: + return "vnode-merge"; + case EVA_TASK_COMPACT: + return "vnode-compact"; + case EVA_TASK_RETENTION: + return "vnode-retention"; + default: + return "unknown"; + } } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 28d27b8893..7fd3f9a8b6 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -389,8 +389,7 @@ int vnodeAsyncCommit(SVnode *pVnode) { TSDB_CHECK_CODE(code, lino, _exit); // schedule the task - code = - vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask); + code = vnodeAsync(COMMIT_TASK_ASYNC, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask); TSDB_CHECK_CODE(code, lino, _exit); _exit: diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 9d326defdd..ff537ef4a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -20,14 +20,13 @@ static volatile int32_t VINIT = 0; -int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { +int vnodeInit(StopDnodeFp stopDnodeFp) { if (atomic_val_compare_exchange_32(&VINIT, 0, 1)) { return 0; } - TAOS_CHECK_RETURN(vnodeAsyncOpen(nthreads)); + TAOS_CHECK_RETURN(vnodeAsyncOpen()); TAOS_CHECK_RETURN(walInit(stopDnodeFp)); - TAOS_CHECK_RETURN(tsdbInit()); monInitVnode(); @@ -36,7 +35,6 @@ int vnodeInit(int nthreads, StopDnodeFp stopDnodeFp) { void vnodeCleanup() { if (atomic_val_compare_exchange_32(&VINIT, 1, 0) == 0) return; - tsdbCleanUp(); vnodeAsyncClose(); walCleanUp(); smaCleanUp(); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 945b4cbeae..6de5298728 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -438,11 +438,6 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC (void)taosThreadMutexInit(&pVnode->mutex, NULL); (void)taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) { - vError("vgId:%d, failed to init commit channel", TD_VID(pVnode)); - goto _err; - } - int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool @@ -558,10 +553,6 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { vnodeAWait(&pVnode->commitTask); - if (vnodeAChannelDestroy(&pVnode->commitChannel, true) != 0) { - vError("vgId:%d, failed to destroy commit channel", TD_VID(pVnode)); - } - vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); tqClose(pVnode->pTq); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 0c11083367..9e3c6861c0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -597,13 +597,11 @@ extern void tsdbEnableBgTask(STsdb *pTsdb); static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) { TAOS_CHECK_RETURN(tsdbDisableAndCancelAllBgTask(pVnode->pTsdb)); TAOS_CHECK_RETURN(vnodeSyncCommit(pVnode)); - TAOS_CHECK_RETURN(vnodeAChannelDestroy(&pVnode->commitChannel, true)); return 0; } static int32_t vnodeEnableBgTask(SVnode *pVnode) { tsdbEnableBgTask(pVnode->pTsdb); - TAOS_CHECK_RETURN(vnodeAChannelInit(1, &pVnode->commitChannel)); return 0; } diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index 179781c2c9..3bdb0a9b1d 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -11,6 +11,6 @@ target_link_libraries( PRIVATE os util transport qcom nodes ) -# if(${BUILD_TEST}) +#if(${BUILD_TEST}) # ADD_SUBDIRECTORY(test) -# endif(${BUILD_TEST}) +#endif(${BUILD_TEST}) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 25c82b8452..7b0504504d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -162,7 +162,7 @@ void ctgTestInitLogFile() { (void)ctgdEnableDebug("cache", true); (void)ctgdEnableDebug("lock", true); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, 1, false) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); ASSERT(0); } diff --git a/source/libs/executor/src/forecastoperator.c b/source/libs/executor/src/forecastoperator.c index a56b0dd214..2985e5e000 100644 --- a/source/libs/executor/src/forecastoperator.c +++ b/source/libs/executor/src/forecastoperator.c @@ -72,17 +72,20 @@ static FORCE_INLINE int32_t forecastEnsureBlockCapacity(SSDataBlock* pBlock, int return TSDB_CODE_SUCCESS; } -static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) { - if (pSupp->cachedRows > ANAL_FORECAST_MAX_ROWS) { - return TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; - } - - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; +static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock, const char* id) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SAnalyticBuf* pBuf = &pSupp->analBuf; - qDebug("block:%d, %p rows:%" PRId64, pSupp->numOfBlocks, pBlock, pBlock->info.rows); + if (pSupp->cachedRows > ANAL_FORECAST_MAX_ROWS) { + code = TSDB_CODE_ANA_ANODE_TOO_MANY_ROWS; + qError("%s rows:%" PRId64 " for forecast cache, error happens, code:%s, upper limit:%d", id, pSupp->cachedRows, + tstrerror(code), ANAL_FORECAST_MAX_ROWS); + return code; + } + pSupp->numOfBlocks++; + qDebug("%s block:%d, %p rows:%" PRId64, id, pSupp->numOfBlocks, pBlock, pBlock->info.rows); for (int32_t j = 0; j < pBlock->info.rows; ++j) { SColumnInfoData* pValCol = taosArrayGet(pBlock->pDataBlock, pSupp->inputValSlot); @@ -98,10 +101,16 @@ static int32_t forecastCacheBlock(SForecastSupp* pSupp, SSDataBlock* pBlock) { pSupp->numOfRows++; code = taosAnalBufWriteColData(pBuf, 0, TSDB_DATA_TYPE_TIMESTAMP, &ts); - if (TSDB_CODE_SUCCESS != code) return code; + if (TSDB_CODE_SUCCESS != code) { + qError("%s failed to write ts in buf, code:%s", id, tstrerror(code)); + return code; + } code = taosAnalBufWriteColData(pBuf, 1, valType, val); - if (TSDB_CODE_SUCCESS != code) return code; + if (TSDB_CODE_SUCCESS != code) { + qError("%s failed to write val in buf, code:%s", id, tstrerror(code)); + return code; + } } return 0; @@ -394,7 +403,7 @@ static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pSupp->cachedRows += pBlock->info.rows; qDebug("%s group:%" PRId64 ", blocks:%d, rows:%" PRId64 ", total rows:%" PRId64, pId, pSupp->groupId, numOfBlocks, pBlock->info.rows, pSupp->cachedRows); - code = forecastCacheBlock(pSupp, pBlock); + code = forecastCacheBlock(pSupp, pBlock, pId); QUERY_CHECK_CODE(code, lino, _end); } else { qDebug("%s group:%" PRId64 ", read finish for new group coming, blocks:%d", pId, pSupp->groupId, numOfBlocks); @@ -405,7 +414,7 @@ static int32_t forecastNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pSupp->cachedRows = pBlock->info.rows; qDebug("%s group:%" PRId64 ", new group, rows:%" PRId64 ", total rows:%" PRId64, pId, pSupp->groupId, pBlock->info.rows, pSupp->cachedRows); - code = forecastCacheBlock(pSupp, pBlock); + code = forecastCacheBlock(pSupp, pBlock, pId); QUERY_CHECK_CODE(code, lino, _end); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 4efa8764e5..b057194cdb 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -668,8 +668,8 @@ int32_t encodeUdfCallRequest(void **buf, const SUdfCallRequest *call) { len += tEncodeDataBlock(buf, &call->block); len += encodeUdfInterBuf(buf, &call->interBuf); } else if (call->callType == TSDB_UDF_CALL_AGG_MERGE) { - len += encodeUdfInterBuf(buf, &call->interBuf); - len += encodeUdfInterBuf(buf, &call->interBuf2); + // len += encodeUdfInterBuf(buf, &call->interBuf); + // len += encodeUdfInterBuf(buf, &call->interBuf2); } else if (call->callType == TSDB_UDF_CALL_AGG_FIN) { len += encodeUdfInterBuf(buf, &call->interBuf); } @@ -690,10 +690,10 @@ void *decodeUdfCallRequest(const void *buf, SUdfCallRequest *call) { buf = tDecodeDataBlock(buf, &call->block); buf = decodeUdfInterBuf(buf, &call->interBuf); break; - case TSDB_UDF_CALL_AGG_MERGE: - buf = decodeUdfInterBuf(buf, &call->interBuf); - buf = decodeUdfInterBuf(buf, &call->interBuf2); - break; + // case TSDB_UDF_CALL_AGG_MERGE: + // buf = decodeUdfInterBuf(buf, &call->interBuf); + // buf = decodeUdfInterBuf(buf, &call->interBuf2); + // break; case TSDB_UDF_CALL_AGG_FIN: buf = decodeUdfInterBuf(buf, &call->interBuf); break; @@ -779,9 +779,9 @@ int32_t encodeUdfCallResponse(void **buf, const SUdfCallResponse *callRsp) { case TSDB_UDF_CALL_AGG_PROC: len += encodeUdfInterBuf(buf, &callRsp->resultBuf); break; - case TSDB_UDF_CALL_AGG_MERGE: - len += encodeUdfInterBuf(buf, &callRsp->resultBuf); - break; + // case TSDB_UDF_CALL_AGG_MERGE: + // len += encodeUdfInterBuf(buf, &callRsp->resultBuf); + // break; case TSDB_UDF_CALL_AGG_FIN: len += encodeUdfInterBuf(buf, &callRsp->resultBuf); break; @@ -801,9 +801,9 @@ void *decodeUdfCallResponse(const void *buf, SUdfCallResponse *callRsp) { case TSDB_UDF_CALL_AGG_PROC: buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); break; - case TSDB_UDF_CALL_AGG_MERGE: - buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); - break; + // case TSDB_UDF_CALL_AGG_MERGE: + // buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); + // break; case TSDB_UDF_CALL_AGG_FIN: buf = decodeUdfInterBuf(buf, &callRsp->resultBuf); break; @@ -1129,8 +1129,9 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SSDataBlock *output, SUdfInterBuf *newState); int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); -int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, - SUdfInterBuf *resultBuf); +// udf todo: aggmerge +// int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, +// SUdfInterBuf *resultBuf); int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output); int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, SScalarParam *output); @@ -2176,11 +2177,11 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf req->interBuf = *state; break; } - case TSDB_UDF_CALL_AGG_MERGE: { - req->interBuf = *state; - req->interBuf2 = *state2; - break; - } + // case TSDB_UDF_CALL_AGG_MERGE: { + // req->interBuf = *state; + // req->interBuf2 = *state2; + // break; + // } case TSDB_UDF_CALL_AGG_FIN: { req->interBuf = *state; break; @@ -2205,10 +2206,10 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf *newState = rsp->resultBuf; break; } - case TSDB_UDF_CALL_AGG_MERGE: { - *newState = rsp->resultBuf; - break; - } + // case TSDB_UDF_CALL_AGG_MERGE: { + // *newState = rsp->resultBuf; + // break; + // } case TSDB_UDF_CALL_AGG_FIN: { *newState = rsp->resultBuf; break; @@ -2241,12 +2242,13 @@ int32_t doCallUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInter // input: interbuf1, interbuf2 // output: resultBuf -int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, - SUdfInterBuf *resultBuf) { - int8_t callType = TSDB_UDF_CALL_AGG_MERGE; - int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); - return err; -} +// udf todo: aggmerge +// int32_t doCallUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, +// SUdfInterBuf *resultBuf) { +// int8_t callType = TSDB_UDF_CALL_AGG_MERGE; +// int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); +// return err; +// } // input: interBuf // output: resultData diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index bbfd43d5f7..ecb24fc77a 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -194,17 +194,17 @@ int32_t udfdCPluginUdfAggProc(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdf } } -int32_t udfdCPluginUdfAggMerge(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf, - void *udfCtx) { - TAOS_UDF_CHECK_PTR_RCODE(inputBuf1, inputBuf2, outputBuf, udfCtx); - SUdfCPluginCtx *ctx = udfCtx; - if (ctx->aggMergeFunc) { - return ctx->aggMergeFunc(inputBuf1, inputBuf2, outputBuf); - } else { - fnError("udfd c plugin aggregation merge not implemented"); - return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; - } -} +// int32_t udfdCPluginUdfAggMerge(SUdfInterBuf *inputBuf1, SUdfInterBuf *inputBuf2, SUdfInterBuf *outputBuf, +// void *udfCtx) { +// TAOS_UDF_CHECK_PTR_RCODE(inputBuf1, inputBuf2, outputBuf, udfCtx); +// SUdfCPluginCtx *ctx = udfCtx; +// if (ctx->aggMergeFunc) { +// return ctx->aggMergeFunc(inputBuf1, inputBuf2, outputBuf); +// } else { +// fnError("udfd c plugin aggregation merge not implemented"); +// return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; +// } +// } int32_t udfdCPluginUdfAggFinish(SUdfInterBuf *buf, SUdfInterBuf *resultData, void *udfCtx) { TAOS_UDF_CHECK_PTR_RCODE(buf, resultData, udfCtx); @@ -378,7 +378,7 @@ int32_t udfdInitializeCPlugin(SUdfScriptPlugin *plugin) { plugin->udfScalarProcFunc = udfdCPluginUdfScalarProc; plugin->udfAggStartFunc = udfdCPluginUdfAggStart; plugin->udfAggProcFunc = udfdCPluginUdfAggProc; - plugin->udfAggMergeFunc = udfdCPluginUdfAggMerge; + // plugin->udfAggMergeFunc = udfdCPluginUdfAggMerge; plugin->udfAggFinishFunc = udfdCPluginUdfAggFinish; SScriptUdfEnvItem items[1] = {{"LD_LIBRARY_PATH", tsUdfdLdLibPath}}; @@ -889,19 +889,19 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { break; } - case TSDB_UDF_CALL_AGG_MERGE: { - SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - if (outBuf.buf != NULL) { - code = udf->scriptPlugin->udfAggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf, udf->scriptUdfCtx); - freeUdfInterBuf(&call->interBuf); - freeUdfInterBuf(&call->interBuf2); - subRsp->resultBuf = outBuf; - } else { - code = terrno; - } - - break; - } + // case TSDB_UDF_CALL_AGG_MERGE: { + // SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; + // if (outBuf.buf != NULL) { + // code = udf->scriptPlugin->udfAggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf, udf->scriptUdfCtx); + // freeUdfInterBuf(&call->interBuf); + // freeUdfInterBuf(&call->interBuf2); + // subRsp->resultBuf = outBuf; + // } else { + // code = terrno; + // } + // + // break; + // } case TSDB_UDF_CALL_AGG_FIN: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; if (outBuf.buf != NULL) { @@ -959,10 +959,10 @@ _exit: freeUdfInterBuf(&subRsp->resultBuf); break; } - case TSDB_UDF_CALL_AGG_MERGE: { - freeUdfInterBuf(&subRsp->resultBuf); - break; - } + // case TSDB_UDF_CALL_AGG_MERGE: { + // freeUdfInterBuf(&subRsp->resultBuf); + // break; + // } case TSDB_UDF_CALL_AGG_FIN: { freeUdfInterBuf(&subRsp->resultBuf); break; @@ -1667,7 +1667,6 @@ static int32_t udfdGlobalDataInit() { } static void udfdGlobalDataDeinit() { - taosHashCleanup(global.udfsHash); uv_mutex_destroy(&global.udfsMutex); uv_mutex_destroy(&global.scriptPluginsMutex); taosMemoryFreeClear(global.loop); @@ -1720,8 +1719,11 @@ void udfdDeinitResidentFuncs() { SUdf **udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); if (udfInHash) { SUdf *udf = *udfInHash; - int32_t code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx); - fnDebug("udfd destroy function returns %d", code); + int32_t code = 0; + if (udf->scriptPlugin->udfDestroyFunc) { + code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx); + fnDebug("udfd %s destroy function returns %d", funcName, code); + } if(taosHashRemove(global.udfsHash, funcName, strlen(funcName)) != 0) { fnError("udfd remove resident function %s failed", funcName); @@ -1729,6 +1731,7 @@ void udfdDeinitResidentFuncs() { taosMemoryFree(udf); } } + taosHashCleanup(global.udfsHash); taosArrayDestroy(global.residentFuncs); fnInfo("udfd resident functions are deinit"); } @@ -1838,15 +1841,15 @@ int main(int argc, char *argv[]) { fnInfo("udfd exit normally"); removeListeningPipe(); - udfdDeinitScriptPlugins(); _exit: - if (globalDataInited) { - udfdGlobalDataDeinit(); - } if (residentFuncsInited) { udfdDeinitResidentFuncs(); } + udfdDeinitScriptPlugins(); + if (globalDataInited) { + udfdGlobalDataDeinit(); + } if (udfSourceDirInited) { udfdDestroyUdfSourceDir(); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index a13472620b..fa656667af 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -116,7 +116,7 @@ static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); (void)strdequote(pPassword); if (strtrim(pPassword) <= 0) { - pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY); } else if (invalidPassword(pPassword)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_PASSWD); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 502dbb57dd..ed1f498a32 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -101,10 +101,13 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c return buildInvalidOperationMsg(pMsgBuf, msg1); } } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, pTableName->z, pTableName->n); + int32_t tbLen = strdequote(tbname); + if (tbLen >= TSDB_TABLE_NAME_LEN) { return buildInvalidOperationMsg(pMsgBuf, msg1); } - if (pTableName->n == 0) { + if (tbLen == 0) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "invalid table name"); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index feaa6910f2..342bd6d66e 100755 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -11411,7 +11411,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm if (pSelect->hasInterpFunc) { // Temporary code - if (pStmt->pOptions->triggerType != STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { + if (tsStreamCoverage == false && pStmt->pOptions->triggerType != STREAM_TRIGGER_FORCE_WINDOW_CLOSE) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Stream interp function only support force window close"); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 9706644324..0cda428487 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -57,8 +57,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Invalid tag name: %s"; case TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG: return "Name or password too long"; - case TSDB_CODE_PAR_PASSWD_EMPTY: - return "Password can not be empty"; + case TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY: + return "Password too short or empty"; case TSDB_CODE_PAR_INVALID_PORT: return "Port should be an integer that is less than 65535 and greater than 0"; case TSDB_CODE_PAR_INVALID_ENDPOINT: diff --git a/source/libs/qworker/src/qwMem.c b/source/libs/qworker/src/qwMem.c index d625bb113a..69d4093221 100644 --- a/source/libs/qworker/src/qwMem.c +++ b/source/libs/qworker/src/qwMem.c @@ -153,6 +153,7 @@ int32_t qwRetrieveJobInfo(QW_FPARAMS_DEF, SQWJobInfo** ppJob) { if (atomic_load_8(&pJob->destroyed)) { QW_UNLOCK(QW_READ, &pJob->lock); + taosHashRelease(gQueryMgmt.pJobInfo, pJob); continue; } diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 2809f05590..3d41b6ef5e 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -258,7 +258,7 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { } } - atomic_add_fetch_64(&gQueryMgmt.stat.taskInitNum, 1); + (void)atomic_add_fetch_64(&gQueryMgmt.stat.taskInitNum, 1); if (acquire && ctx) { QW_RET(qwAcquireTaskCtx(QW_FPARAMS(), ctx)); @@ -283,7 +283,7 @@ void qwFreeTaskHandle(SQWTaskCtx *ctx) { qDestroyTask(otaskHandle); taosDisableMemPoolUsage(); - atomic_add_fetch_64(&gQueryMgmt.stat.taskExecDestroyNum, 1); + (void)atomic_add_fetch_64(&gQueryMgmt.stat.taskExecDestroyNum, 1); qDebug("task handle destroyed"); } @@ -297,7 +297,7 @@ void qwFreeSinkHandle(SQWTaskCtx *ctx) { dsDestroyDataSinker(osinkHandle); QW_SINK_DISABLE_MEMPOOL(); - atomic_add_fetch_64(&gQueryMgmt.stat.taskSinkDestroyNum, 1); + (void)atomic_add_fetch_64(&gQueryMgmt.stat.taskSinkDestroyNum, 1); qDebug("sink handle destroyed"); } @@ -409,6 +409,8 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { if (ctx->pJobInfo && TSDB_CODE_SUCCESS != ctx->pJobInfo->errCode) { QW_UPDATE_RSP_CODE(ctx, ctx->pJobInfo->errCode); + } else { + QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_TSC_QUERY_CANCELLED); } atomic_store_ptr(&ctx->taskHandle, NULL); @@ -428,7 +430,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { QW_TASK_DLOG_E("task ctx dropped"); - atomic_add_fetch_64(&gQueryMgmt.stat.taskDestroyNum, 1); + (void)atomic_add_fetch_64(&gQueryMgmt.stat.taskDestroyNum, 1); return code; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 641fa03f7a..4a9eea66e2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -449,51 +449,64 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes return TSDB_CODE_SUCCESS; } -int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32_t code) { - if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - void *rsp = NULL; - int32_t dataLen = 0; - int32_t rawLen = 0; - SOutputData sOutput = {0}; - if (TSDB_CODE_SUCCESS == code) { - code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); +int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWMsg *qwMsg, int32_t code) { + if (QUERY_RSP_POLICY_QUICK != tsQueryRspPolicy) { + return TSDB_CODE_SUCCESS; + } + + SQWTaskCtx *ctx = NULL; + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + + if (!QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { + goto _return; + } + + void *rsp = NULL; + int32_t dataLen = 0; + int32_t rawLen = 0; + SOutputData sOutput = {0}; + if (TSDB_CODE_SUCCESS == code) { + code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); + } + + if (code) { + qwFreeFetchRsp(rsp); + rsp = NULL; + dataLen = 0; + } + + if (NULL == rsp && TSDB_CODE_SUCCESS == code) { + goto _return; + } + + if (NULL != rsp) { + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); + if (qComplete) { + atomic_store_8((int8_t *)&ctx->queryEnd, true); + if (!ctx->dynamicTask) { + qwFreeSinkHandle(ctx); } - - if (code) { - qwFreeFetchRsp(rsp); - rsp = NULL; - dataLen = 0; - } - - if (NULL == rsp && TSDB_CODE_SUCCESS == code) { - return TSDB_CODE_SUCCESS; - } - - if (NULL != rsp) { - bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - - qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); - if (qComplete) { - atomic_store_8((int8_t *)&ctx->queryEnd, true); - if (!ctx->dynamicTask) { - qwFreeSinkHandle(ctx); - } - } - } - - qwMsg->connInfo = ctx->dataConnInfo; - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - QW_ERR_RET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); - rsp = NULL; - - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); } } - return TSDB_CODE_SUCCESS; + qwMsg->connInfo = ctx->dataConnInfo; + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + QW_ERR_JRET(qwBuildAndSendFetchRsp(ctx, ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code)); + rsp = NULL; + + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); + +_return: + + if (ctx) { + qwReleaseTaskCtx(mgmt, ctx); + } + + return code; } int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg) { @@ -748,6 +761,13 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); + QW_LOCK(QW_WRITE, &ctx->lock); + + if (QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP) || QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { + QW_TASK_WLOG("task dropping or already dropped, drop event:%d", QW_GET_EVENT(ctx, QW_EVENT_DROP)); + QW_ERR_JRET(ctx->rspCode); + } + ctx->ctrlConnInfo = qwMsg->connInfo; ctx->sId = sId; ctx->phase = -1; @@ -767,7 +787,8 @@ _return: if (code) { (void)qwDropTask(QW_FPARAMS()); } - + + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } @@ -848,7 +869,7 @@ _return: input.msgType = qwMsg->msgType; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code)); + QW_ERR_RET(qwQuickRspFetchReq(QW_FPARAMS(), qwMsg, code)); QW_RET(TSDB_CODE_SUCCESS); } @@ -1028,12 +1049,16 @@ _return: if (code || rsp) { bool rsped = false; + + ctx = NULL; + (void)qwAcquireTaskCtx(QW_FPARAMS(), &ctx); + if (ctx) { qwDbgSimulateRedirect(qwMsg, ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); } - if (!rsped) { + if (!rsped && ctx) { code = qwBuildAndSendFetchRsp(ctx, qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); if (TSDB_CODE_SUCCESS != code) { QW_TASK_ELOG("fetch rsp send fail, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", @@ -1046,6 +1071,8 @@ _return: qwFreeFetchRsp(rsp); rsp = NULL; } + + qwReleaseTaskCtx(mgmt, ctx); } else { // qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); } diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index c25b0f34fc..41ac0117f3 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -250,6 +250,9 @@ void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId); // inject stream errors void chkptFailedByRetrieveReqToSource(SStreamTask* pTask, int64_t checkpointId); +int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type); +int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 876cd1b472..0ec66cd2ce 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -21,7 +21,9 @@ static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName); static int32_t deleteCheckpointFile(const char* id, const char* name); static int32_t streamTaskUploadCheckpoint(const char* id, const char* path, int64_t checkpointId); +#ifdef BUILD_NO_CALL static int32_t deleteCheckpoint(const char* id); +#endif static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask); static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, @@ -998,7 +1000,7 @@ static int32_t doFindNotSendUpstream(SStreamTask* pTask, SArray* pList, SArray** return 0; } -static int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList) { +int32_t chkptTriggerRecvMonitorHelper(SStreamTask* pTask, void* param, SArray** ppNotSendList) { const char* id = pTask->id.idStr; SArray* pList = pTask->upstreamInfo.pList; // send msg to retrieve checkpoint trigger msg SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; @@ -1492,6 +1494,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path, int64_t che return 0; } +#ifdef BUILD_NO_CALL int32_t deleteCheckpoint(const char* id) { if (id == NULL || strlen(id) == 0) { stError("deleteCheckpoint parameters invalid"); @@ -1504,6 +1507,7 @@ int32_t deleteCheckpoint(const char* id) { } return 0; } +#endif int32_t deleteCheckpointFile(const char* id, const char* name) { char object[128] = {0}; diff --git a/source/libs/stream/test/streamCheckPointTest.cpp b/source/libs/stream/test/streamCheckPointTest.cpp new file mode 100644 index 0000000000..80dd3ec142 --- /dev/null +++ b/source/libs/stream/test/streamCheckPointTest.cpp @@ -0,0 +1,270 @@ +#include +#include "tstream.h" +#include "streamInt.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +void initTaskLock(SStreamTask* pTask) { + TdThreadMutexAttr attr = {0}; + int32_t code = taosThreadMutexAttrInit(&attr); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + code = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + code = taosThreadMutexInit(&pTask->lock, &attr); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + code = taosThreadMutexAttrDestroy(&attr); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); +} + +TEST(streamCheckpointTest, StreamTaskProcessCheckpointTriggerRsp) { + SStreamTask* pTask = NULL; + int64_t uid = 1111111111111111; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 123111; + pTask->chkInfo.pActiveInfo->transId = 4561111; + + streamTaskSetStatusReady(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SCheckpointTriggerRsp pRsp; + memset(&pRsp, 0, sizeof(SCheckpointTriggerRsp)); + pRsp.rspCode = TSDB_CODE_SUCCESS; + pRsp.checkpointId = 123; + pRsp.transId = 456; + pRsp.upstreamTaskId = 789; + + code = streamTaskProcessCheckpointTriggerRsp(pTask, &pRsp); + ASSERT_NE(code, TSDB_CODE_SUCCESS); + + pRsp.rspCode = TSDB_CODE_FAILED; + code = streamTaskProcessCheckpointTriggerRsp(pTask, &pRsp); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + tFreeStreamTask(pTask); + taosArrayDestroy(array); +} + +TEST(streamCheckpointTest, StreamTaskSetFailedCheckpointId) { + SStreamTask* pTask = NULL; + int64_t uid = 1111111111111111; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + pInfo->failedId = 0; + + int64_t failedCheckpointId = 123; + + streamTaskSetFailedCheckpointId(pTask, failedCheckpointId); + ASSERT_EQ(pInfo->failedId, failedCheckpointId); + + streamTaskSetFailedCheckpointId(pTask, 0); + ASSERT_EQ(pInfo->failedId, failedCheckpointId); + + streamTaskSetFailedCheckpointId(pTask, pInfo->failedId - 1); + ASSERT_EQ(pInfo->failedId, failedCheckpointId); + tFreeStreamTask(pTask); + taosArrayDestroy(array); +} + +TEST(UploadCheckpointDataTest, UploadSuccess) { + streamMetaInit(); + SStreamTask* pTask = NULL; + int64_t uid = 1111111111111111; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + int64_t checkpointId = 123; + int64_t dbRefId = 1; + ECHECKPOINT_BACKUP_TYPE type = DATA_UPLOAD_S3; + + STaskDbWrapper* pBackend = NULL; + int64_t processVer = -1; + const char *path = "/tmp/backend3/stream"; + code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SStreamState *pState = streamStateOpen((char *)path, pTask, 0, 0); + ASSERT(pState != NULL); + + pTask->pBackend = pState->pTdbState->pOwner->pBackend; + + code = taskDbDoCheckpoint(pTask->pBackend, checkpointId, 0); + ASSERT(code == 0); + + int32_t result = uploadCheckpointData(pTask, checkpointId, dbRefId, type); + + EXPECT_EQ(result, TSDB_CODE_SUCCESS) << "uploadCheckpointData should return 0 on success"; + tFreeStreamTask(pTask); + taosRemoveDir(path); + streamStateClose(pState, true); + taosArrayDestroy(array); +} + +TEST(UploadCheckpointDataTest, UploadDisabled) { + SStreamTask* pTask = NULL; + int64_t uid = 2222222222222; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + int64_t checkpointId = 123; + int64_t dbRefId = 1; + + STaskDbWrapper* pBackend = NULL; + int64_t processVer = -1; + const char *path = "/tmp/backend4/stream"; + code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + SStreamState *pState = streamStateOpen((char *)path, pTask, 0, 0); + ASSERT(pState != NULL); + + pTask->pBackend = pState->pTdbState->pOwner->pBackend; + + code = taskDbDoCheckpoint(pTask->pBackend, checkpointId, 0); + ASSERT(code == 0); + + ECHECKPOINT_BACKUP_TYPE type = DATA_UPLOAD_DISABLE; + + int32_t result = uploadCheckpointData(pTask, checkpointId, dbRefId, type); + + EXPECT_NE(result, TSDB_CODE_SUCCESS) << "uploadCheckpointData should return 0 when backup type is disabled"; + + streamStateClose(pState, true); + tFreeStreamTask(pTask); + taosArrayDestroy(array); +} + +TEST(StreamTaskAlreadySendTriggerTest, AlreadySendTrigger) { + SStreamTask* pTask = NULL; + int64_t uid = 2222222222222; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 123111; + pTask->chkInfo.pActiveInfo->transId = 4561111; + + streamTaskSetStatusReady(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + int32_t downstreamNodeId = 1; + int64_t sendingCheckpointId = 123; + TSKEY ts = taosGetTimestampMs(); + + STaskTriggerSendInfo triggerInfo; + triggerInfo.sendTs = ts; + triggerInfo.recved = false; + triggerInfo.nodeId = downstreamNodeId; + + taosArrayPush(pTask->chkInfo.pActiveInfo->pDispatchTriggerList, &triggerInfo); + + pTask->chkInfo.pActiveInfo->dispatchTrigger = true; + bool result = streamTaskAlreadySendTrigger(pTask, downstreamNodeId); + + EXPECT_TRUE(result) << "The trigger message should have been sent to the downstream node"; + + tFreeStreamTask(pTask); + taosArrayDestroy(array); +} + +TEST(ChkptTriggerRecvMonitorHelperTest, chkptTriggerRecvMonitorHelper) { + SStreamTask* pTask = NULL; + int64_t uid = 2222222222222; + SArray* array = taosArrayInit(4, POINTER_BYTES); + int32_t code = tNewStreamTask(uid, TASK_LEVEL__SINK, NULL, false, 0, 0, array, + false, 1, &pTask); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + initTaskLock(pTask); + + const char *path = "/tmp/backend5/stream"; + code = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL, &pTask->pMeta); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + code = streamTaskCreateActiveChkptInfo(&pTask->chkInfo.pActiveInfo); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->chkInfo.pActiveInfo->activeId = 123111; + pTask->chkInfo.pActiveInfo->chkptTriggerMsgTmr.launchChkptId = pTask->chkInfo.pActiveInfo->activeId; + pTask->chkInfo.pActiveInfo->transId = 4561111; + pTask->chkInfo.startTs = 11111; + + streamTaskSetStatusReady(pTask); + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + + int32_t downstreamNodeId = 1; + int64_t sendingCheckpointId = 123; + TSKEY ts = taosGetTimestampMs(); + + STaskTriggerSendInfo triggerInfo; + triggerInfo.sendTs = ts; + triggerInfo.recved = false; + triggerInfo.nodeId = downstreamNodeId; + + taosArrayPush(pTask->chkInfo.pActiveInfo->pDispatchTriggerList, &triggerInfo); + + pTask->chkInfo.pActiveInfo->dispatchTrigger = true; + SArray* array1 = NULL; + code = chkptTriggerRecvMonitorHelper(pTask, NULL, &array1); + EXPECT_EQ(code, TSDB_CODE_SUCCESS); + + pTask->pMeta->fatalInfo.code = TSDB_CODE_SUCCESS; + streamSetFatalError(pTask->pMeta, code, __func__, __LINE__); + + pTask->pMeta->fatalInfo.code = TSDB_CODE_FAILED; + streamSetFatalError(pTask->pMeta, code, __func__, __LINE__); + tFreeStreamTask(pTask); + taosArrayDestroy(array); + taosArrayDestroy(array1); +} diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 2333a4a6a2..19a3f211b1 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1491,7 +1491,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, ofpCell = tdbPageGetCell(ofp, 0); int lastKeyPage = 0; - if (nLeftKey <= maxLocal - sizeof(SPgno)) { + if (nLeftKey <= ofp->maxLocal - sizeof(SPgno)) { bytes = nLeftKey; lastKeyPage = 1; lastKeyPageSpace = ofp->maxLocal - sizeof(SPgno) - nLeftKey; diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 5da2307cb3..d82c56c99a 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -355,7 +355,7 @@ void atomic_store_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return; + if (ret_num.i == old_num.i) break; } } @@ -414,6 +414,8 @@ int64_t atomic_exchange_64(int64_t volatile* ptr, int64_t val) { } double atomic_exchange_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -425,9 +427,11 @@ double atomic_exchange_double(double volatile* ptr, double val) { ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); if (ret_num.i == old_num.i) { - return ret_num.d; + ret = ret_num.d; + break; } } + return ret; } void* atomic_exchange_ptr(void* ptr, void* val) { @@ -589,6 +593,8 @@ int64_t atomic_fetch_add_64(int64_t volatile* ptr, int64_t val) { } double atomic_fetch_add_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -599,8 +605,13 @@ double atomic_fetch_add_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) { + ret = ret_num.d; + break; + } } + + return ret; } void* atomic_fetch_add_ptr(void* ptr, int64_t val) { @@ -710,6 +721,8 @@ int64_t atomic_fetch_sub_64(int64_t volatile* ptr, int64_t val) { } double atomic_fetch_sub_double(double volatile* ptr, double val) { + double ret = 0; + for (;;) { double_number old_num = {0}; old_num.d = *ptr; // current old value @@ -720,8 +733,13 @@ double atomic_fetch_sub_double(double volatile* ptr, double val) { double_number ret_num = {0}; ret_num.i = atomic_val_compare_exchange_64((volatile int64_t*)ptr, old_num.i, new_num.i); - if (ret_num.i == old_num.i) return ret_num.d; + if (ret_num.i == old_num.i) { + ret = ret_num.d; + break; + } } + + return ret; } void* atomic_fetch_sub_ptr(void* ptr, int64_t val) { diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index f888835d95..603f5da3f7 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -23,18 +23,16 @@ int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start) int32_t code = pthread_create(tid, attr, start, arg); if (code) { taosThreadClear(tid); - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { +int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); - int32_t code = pthread_attr_destroy(attr); + int32_t code = pthread_attr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -44,8 +42,7 @@ int32_t taosThreadAttrGetDetachState(const TdThreadAttr *attr, int32_t *detachst OS_PARAM_CHECK(detachstate); int32_t code = pthread_attr_getdetachstate(attr, detachstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -55,8 +52,7 @@ int32_t taosThreadAttrGetInheritSched(const TdThreadAttr *attr, int32_t *inherit OS_PARAM_CHECK(inheritsched); int32_t code = pthread_attr_getinheritsched(attr, inheritsched); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -66,8 +62,7 @@ int32_t taosThreadAttrGetSchedParam(const TdThreadAttr *attr, struct sched_param OS_PARAM_CHECK(param); int32_t code = pthread_attr_getschedparam(attr, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -77,8 +72,7 @@ int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr *attr, int32_t *policy) OS_PARAM_CHECK(policy); int32_t code = pthread_attr_getschedpolicy(attr, policy); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -88,8 +82,7 @@ int32_t taosThreadAttrGetScope(const TdThreadAttr *attr, int32_t *contentionscop OS_PARAM_CHECK(contentionscope); int32_t code = pthread_attr_getscope(attr, contentionscope); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -99,18 +92,16 @@ int32_t taosThreadAttrGetStackSize(const TdThreadAttr *attr, size_t *stacksize) OS_PARAM_CHECK(stacksize); int32_t code = pthread_attr_getstacksize(attr, stacksize); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } int32_t taosThreadAttrInit(TdThreadAttr *attr) { OS_PARAM_CHECK(attr); - int32_t code = pthread_attr_init(attr); + int32_t code = pthread_attr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -119,8 +110,7 @@ int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachstate) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setdetachstate(attr, detachstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -129,8 +119,7 @@ int32_t taosThreadAttrSetInheritSched(TdThreadAttr *attr, int32_t inheritsched) OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setinheritsched(attr, inheritsched); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -139,8 +128,7 @@ int32_t taosThreadAttrSetSchedParam(TdThreadAttr *attr, const struct sched_param OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setschedparam(attr, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -149,8 +137,7 @@ int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr *attr, int32_t policy) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setschedpolicy(attr, policy); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -159,8 +146,7 @@ int32_t taosThreadAttrSetScope(TdThreadAttr *attr, int32_t contentionscope) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setscope(attr, contentionscope); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -169,17 +155,15 @@ int32_t taosThreadAttrSetStackSize(TdThreadAttr *attr, size_t stacksize) { OS_PARAM_CHECK(attr); int32_t code = pthread_attr_setstacksize(attr, stacksize); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadCancel(TdThread thread) { - int32_t code = pthread_cancel(thread); +int32_t taosThreadCancel(TdThread thread) { + int32_t code = pthread_cancel(thread); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -191,8 +175,7 @@ int32_t taosThreadCondDestroy(TdThreadCond *cond) { #else int32_t code = pthread_cond_destroy(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -206,8 +189,7 @@ int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { #else int32_t code = pthread_cond_init(cond, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -221,8 +203,7 @@ int32_t taosThreadCondSignal(TdThreadCond *cond) { #else int32_t code = pthread_cond_signal(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -236,8 +217,7 @@ int32_t taosThreadCondBroadcast(TdThreadCond *cond) { #else int32_t code = pthread_cond_broadcast(cond); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -254,8 +234,7 @@ int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { #else int32_t code = pthread_cond_wait(cond, mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -274,7 +253,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const return TAOS_SYSTEM_WINAPI_ERROR(error); #else int32_t code = pthread_cond_timedwait(cond, mutex, abstime); - if(code == ETIMEDOUT) { + if (code == ETIMEDOUT) { return TSDB_CODE_TIMEOUT_ERROR; } else if (code) { return TAOS_SYSTEM_ERROR(code); @@ -291,14 +270,14 @@ int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif } int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared) { + OS_PARAM_CHECK(attr); OS_PARAM_CHECK(pshared); #ifdef __USE_WIN_THREAD if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE; @@ -307,8 +286,7 @@ int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *psha OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -321,8 +299,7 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -337,8 +314,7 @@ int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) { OS_PARAM_CHECK(attr); int32_t code = pthread_condattr_setclock(attr, clockId); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -351,28 +327,24 @@ int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) { #else int32_t code = pthread_condattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif } -int32_t taosThreadDetach(TdThread thread) { - int32_t code = pthread_detach(thread); +int32_t taosThreadDetach(TdThread thread) { + int32_t code = pthread_detach(thread); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadEqual(TdThread t1, TdThread t2) { - return pthread_equal(t1, t2); -} +int32_t taosThreadEqual(TdThread t1, TdThread t2) { return pthread_equal(t1, t2); } -void taosThreadExit(void *valuePtr) { - if(valuePtr) return pthread_exit(valuePtr); +void taosThreadExit(void *valuePtr) { + if (valuePtr) return pthread_exit(valuePtr); } int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param) { @@ -380,21 +352,17 @@ int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_p OS_PARAM_CHECK(param); int32_t code = pthread_getschedparam(thread, policy, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -void *taosThreadGetSpecific(TdThreadKey key) { - return pthread_getspecific(key); -} +void *taosThreadGetSpecific(TdThreadKey key) { return pthread_getspecific(key); } -int32_t taosThreadJoin(TdThread thread, void **valuePtr) { - int32_t code = pthread_join(thread, valuePtr); +int32_t taosThreadJoin(TdThread thread, void **valuePtr) { + int32_t code = pthread_join(thread, valuePtr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -403,26 +371,23 @@ int32_t taosThreadKeyCreate(TdThreadKey *key, void (*destructor)(void *)) { OS_PARAM_CHECK(key); int32_t code = pthread_key_create(key, destructor); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadKeyDelete(TdThreadKey key) { - int32_t code = pthread_key_delete(key); +int32_t taosThreadKeyDelete(TdThreadKey key) { + int32_t code = pthread_key_delete(key); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadKill(TdThread thread, int32_t sig) { - int32_t code = pthread_kill(thread, sig); +int32_t taosThreadKill(TdThread thread, int32_t sig) { + int32_t code = pthread_kill(thread, sig); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -439,8 +404,7 @@ int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_destroy(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -459,8 +423,7 @@ int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) #else int32_t code = pthread_mutex_init(mutex, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -474,8 +437,7 @@ int32_t taosThreadMutexLock(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_lock(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -507,8 +469,7 @@ int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { #else int32_t code = pthread_mutex_unlock(mutex); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -521,8 +482,7 @@ int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -537,8 +497,7 @@ int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr *attr, int32_t *ps OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -557,8 +516,7 @@ int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr *attr, int32_t *kind) OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_gettype(attr, kind); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -571,8 +529,7 @@ int32_t taosThreadMutexAttrInit(TdThreadMutexAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -585,8 +542,7 @@ int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr *attr, int32_t pshared) OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -603,8 +559,7 @@ int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) { OS_PARAM_CHECK(attr); int32_t code = pthread_mutexattr_settype(attr, kind); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -613,8 +568,7 @@ int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr *attr, int32_t kind) { int32_t taosThreadOnce(TdThreadOnce *onceControl, void (*initRoutine)(void)) { int32_t code = pthread_once(onceControl, initRoutine); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -626,11 +580,10 @@ int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { */ return 0; #else - OS_PARAM_CHECK(rwlock); + OS_PARAM_CHECK(rwlock); int32_t code = pthread_rwlock_destroy(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -645,8 +598,7 @@ int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *a #else int32_t code = pthread_rwlock_init(rwlock, attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -660,8 +612,7 @@ int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_rdlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -683,8 +634,7 @@ int32_t taosThreadRwlockTryRdlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_tryrdlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -699,8 +649,7 @@ int32_t taosThreadRwlockTryWrlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_trywrlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -718,8 +667,7 @@ int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_unlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -734,8 +682,7 @@ int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { #else int32_t code = pthread_rwlock_wrlock(rwlock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -748,14 +695,14 @@ int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_destroy(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif } int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t *pshared) { + OS_PARAM_CHECK(attr); OS_PARAM_CHECK(pshared); #ifdef __USE_WIN_THREAD if (pshared) *pshared = PTHREAD_PROCESS_PRIVATE; @@ -763,8 +710,7 @@ int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr *attr, int32_t * #else int32_t code = pthread_rwlockattr_getpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -777,8 +723,7 @@ int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr *attr) { OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_init(attr); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -791,8 +736,7 @@ int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared OS_PARAM_CHECK(attr); int32_t code = pthread_rwlockattr_setpshared(attr, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -800,20 +744,18 @@ int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr *attr, int32_t pshared TdThread taosThreadSelf(void) { return pthread_self(); } -int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { - int32_t code = pthread_setcancelstate(state, oldstate); +int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { + int32_t code = pthread_setcancelstate(state, oldstate); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { - int32_t code = pthread_setcanceltype(type, oldtype); +int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { + int32_t code = pthread_setcanceltype(type, oldtype); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -822,18 +764,16 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc OS_PARAM_CHECK(param); int32_t code = pthread_setschedparam(thread, policy, param); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } -int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { +int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { OS_PARAM_CHECK(value); - int32_t code = pthread_setspecific(key, value); + int32_t code = pthread_setspecific(key, value); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; } @@ -845,8 +785,7 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_destroy((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -860,8 +799,7 @@ int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #else int32_t code = pthread_spin_init((pthread_spinlock_t *)lock, pshared); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -874,8 +812,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_lock((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif @@ -901,20 +838,17 @@ int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { #else int32_t code = pthread_spin_unlock((pthread_spinlock_t *)lock); if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + return (terrno = TAOS_SYSTEM_ERROR(code)); } return code; #endif } -void taosThreadTestCancel(void) { - return pthread_testcancel(); -} +void taosThreadTestCancel(void) { return pthread_testcancel(); } -void taosThreadClear(TdThread *thread) { +void taosThreadClear(TdThread *thread) { if (!thread) return; - (void)memset(thread, 0, sizeof(TdThread)); + (void)memset(thread, 0, sizeof(TdThread)); } #ifdef WINDOWS diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 29cbcaeb2c..75cb2b91a2 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -31,7 +31,7 @@ #include #include #include -//#define TM_YEAR_BASE 1970 //origin +// #define TM_YEAR_BASE 1970 //origin #define TM_YEAR_BASE 1900 // slguan // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) @@ -345,8 +345,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { #endif } -size_t -taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t){ +size_t taosStrfTime(char *s, size_t maxsize, char const *format, struct tm const *t) { if (!s || !format || !t) return 0; return strftime(s, maxsize, format, t); } @@ -379,7 +378,7 @@ int32_t taosTime(time_t *t) { if (t == NULL) { return TSDB_CODE_INVALID_PARA; } - time_t r = time(t); + time_t r = time(t); if (r == (time_t)-1) { return TAOS_SYSTEM_ERROR(errno); } @@ -433,15 +432,15 @@ time_t taosMktime(struct tm *timep, timezone_t tz) { return result; } int64_t tzw = 0; - #ifdef _MSC_VER - #if _MSC_VER >= 1900 - tzw = _timezone; - #endif - #endif +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + tzw = _timezone; +#endif +#endif return user_mktime64(timep->tm_year + 1900, timep->tm_mon + 1, timep->tm_mday, timep->tm_hour, timep->tm_min, timep->tm_sec, tzw); #else - time_t r = tz != NULL ? mktime_z(tz, timep) : mktime(timep); + time_t r = (tz != NULL ? mktime_z(tz, timep) : mktime(timep)); if (r == (time_t)-1) { terrno = TAOS_SYSTEM_ERROR(errno); } @@ -450,7 +449,7 @@ time_t taosMktime(struct tm *timep, timezone_t tz) { #endif } -struct tm *taosGmTimeR(const time_t *timep, struct tm *result){ +struct tm *taosGmTimeR(const time_t *timep, struct tm *result) { if (timep == NULL || result == NULL) { return NULL; } @@ -461,7 +460,7 @@ struct tm *taosGmTimeR(const time_t *timep, struct tm *result){ #endif } -time_t taosTimeGm(struct tm *tmp){ +time_t taosTimeGm(struct tm *tmp) { if (tmp == NULL) { return -1; } @@ -530,7 +529,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf, int3 } return result; #else - res = tz != NULL ? localtime_rz(tz, timep, result): localtime_r(timep, result); + res = (tz != NULL ? localtime_rz(tz, timep, result) : localtime_r(timep, result)); if (res == NULL && buf != NULL) { (void)snprintf(buf, bufSize, "NaN"); } @@ -544,8 +543,8 @@ int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { int32_t code = 0; #ifdef WINDOWS - LARGE_INTEGER t; - FILETIME f; + LARGE_INTEGER t; + FILETIME f; GetSystemTimeAsFileTime(&f); t.QuadPart = f.dwHighDateTime; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 395c1cbb82..cad3f426f2 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -750,13 +750,14 @@ int32_t taosSetGlobalTimezone(const char *tz) { int32_t code = TSDB_CODE_SUCCESS; uDebug("[tz]set timezone to %s", tz) #ifdef WINDOWS - char winStr[TD_TIMEZONE_LEN * 2] = {0}; + char winStr[TD_TIMEZONE_LEN * 2] = {0}; for (size_t i = 0; i < W_TZ_CITY_NUM; i++) { if (strcmp(tz_win[i][0], tz) == 0) { char keyPath[256] = {0}; char keyValue[100] = {0}; DWORD keyValueSize = sizeof(keyValue); - snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", tz_win[i][1]); + snprintf(keyPath, sizeof(keyPath), "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Time Zones\\%s", + tz_win[i][1]); RegGetValue(HKEY_LOCAL_MACHINE, keyPath, "Display", RRF_RT_ANY, NULL, (PVOID)&keyValue, &keyValueSize); if (keyValueSize > 0) { keyValue[4] = (keyValue[4] == '+' ? '-' : '+'); @@ -770,7 +771,7 @@ int32_t taosSetGlobalTimezone(const char *tz) { _putenv(winStr); _tzset(); #else - code = setenv("TZ", tz, 1); + code = setenv("TZ", tz, 1); if (-1 == code) { terrno = TAOS_SYSTEM_ERROR(errno); return terrno; @@ -779,7 +780,7 @@ int32_t taosSetGlobalTimezone(const char *tz) { tzset(); #endif - time_t tx1 = taosGetTimestampSec(); + time_t tx1 = taosGetTimestampSec(); return taosFormatTimezoneStr(tx1, tz, NULL, tsTimezoneStr); } @@ -797,7 +798,7 @@ int32_t taosGetLocalTimezoneOffset() { #endif } -int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *outTimezoneStr){ +int32_t taosFormatTimezoneStr(time_t t, const char *tz, timezone_t sp, char *outTimezoneStr) { struct tm tm1; if (taosLocalTime(&t, &tm1, NULL, 0, sp) == NULL) { uError("%s failed to get local time: code:%d", __FUNCTION__, errno); @@ -813,16 +814,17 @@ int32_t taosFormatTimezoneStr(time_t t, const char* tz, timezone_t sp, char *out */ char str1[TD_TIMEZONE_LEN] = {0}; - if (taosStrfTime(str1, sizeof(str1), "%Z", &tm1) == 0){ + if (taosStrfTime(str1, sizeof(str1), "%Z", &tm1) == 0) { uError("failed to get timezone name"); return TSDB_CODE_TIME_ERROR; } char str2[TD_TIMEZONE_LEN] = {0}; - if (taosStrfTime(str2, sizeof(str2), "%z", &tm1) == 0){ + if (taosStrfTime(str2, sizeof(str2), "%z", &tm1) == 0) { uError("failed to get timezone offset"); return TSDB_CODE_TIME_ERROR; } + (void)snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (%s, %s)", tz, str1, str2); uDebug("[tz] system timezone:%s", outTimezoneStr); return 0; @@ -847,7 +849,6 @@ void getTimezoneStr(char *tz) { goto END; } while (0); - TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); if (pFile == NULL) { uWarn("[tz] failed to open /etc/timezone, reason:%s", strerror(errno)); @@ -876,8 +877,8 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { char value[100] = {0}; char keyPath[100] = {0}; DWORD bufferSize = sizeof(value); - LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", "TimeZoneKeyName", - RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); + LONG result = RegGetValue(HKEY_LOCAL_MACHINE, "SYSTEM\\CurrentControlSet\\Control\\TimeZoneInformation", + "TimeZoneKeyName", RRF_RT_ANY, NULL, (PVOID)&value, &bufferSize); if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } @@ -891,9 +892,9 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { if (result != ERROR_SUCCESS) { return TAOS_SYSTEM_WINAPI_ERROR(result); } - if (bufferSize > 0) { // value like (UTC+05:30) Chennai, Kolkata, Mumbai, New Delhi - snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, - value[4], value[5], value[6], value[8], value[9]); + if (bufferSize > 0) { // value like (UTC+05:30) Chennai, Kolkata, Mumbai, New Delhi + snprintf(outTimezoneStr, TD_TIMEZONE_LEN, "%s (UTC, %c%c%c%c%c)", outTimezoneStr, value[4], value[5], + value[6], value[8], value[9]); } break; } @@ -903,7 +904,7 @@ int32_t taosGetSystemTimezone(char *outTimezoneStr) { #else char tz[TD_TIMEZONE_LEN] = {0}; getTimezoneStr(tz); - time_t tx1 = taosGetTimestampSec(); + time_t tx1 = taosGetTimestampSec(); return taosFormatTimezoneStr(tx1, tz, NULL, outTimezoneStr); #endif } \ No newline at end of file diff --git a/source/os/test/CMakeLists.txt b/source/os/test/CMakeLists.txt index 13fea463f7..d592168166 100644 --- a/source/os/test/CMakeLists.txt +++ b/source/os/test/CMakeLists.txt @@ -14,20 +14,32 @@ ENDIF() INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) -# osTests -add_executable(osTests "osTests.cpp") -target_link_libraries(osTests os util gtest_main) +if(TD_LINUX) +add_executable(osAtomicTests "osAtomicTests.cpp") +target_link_libraries(osAtomicTests os util gtest_main) add_test( - NAME osTests - COMMAND osTests + NAME osAtomicTests + COMMAND osAtomicTests ) +endif() -add_executable(osSystemTests "osSystemTests.cpp") -target_link_libraries(osSystemTests os util gtest_main) +if(TD_LINUX) +add_executable(osDirTests "osDirTests.cpp") +target_link_libraries(osDirTests os util gtest_main) add_test( - NAME osSystemTests - COMMAND osSystemTests + NAME osDirTests + COMMAND osDirTests ) +endif() + +if(TD_LINUX) +add_executable(osEnvTests "osEnvTests.cpp") +target_link_libraries(osEnvTests os util gtest_main) +add_test( + NAME osEnvTests + COMMAND osEnvTests +) +endif() add_executable(osMathTests "osMathTests.cpp") target_link_libraries(osMathTests os util gtest_main) @@ -36,6 +48,13 @@ add_test( COMMAND osMathTests ) +add_executable(osSemaphoreTests "osSemaphoreTests.cpp") +target_link_libraries(osSemaphoreTests os util gtest_main) +add_test( + NAME osSemaphoreTests + COMMAND osSemaphoreTests +) + add_executable(osSignalTests "osSignalTests.cpp") target_link_libraries(osSignalTests os util gtest_main) add_test( @@ -57,12 +76,28 @@ add_test( COMMAND osStringTests ) +add_executable(osTests "osTests.cpp") +target_link_libraries(osTests os util gtest_main) +add_test( + NAME osTests + COMMAND osTests +) + +add_executable(osSystemTests "osSystemTests.cpp") +target_link_libraries(osSystemTests os util gtest_main) +add_test( + NAME osSystemTests + COMMAND osSystemTests +) + +if(TD_LINUX) add_executable(osThreadTests "osThreadTests.cpp") target_link_libraries(osThreadTests os util gtest_main) add_test( NAME osThreadTests COMMAND osThreadTests ) +endif() add_executable(osTimeTests "osTimeTests.cpp") target_link_libraries(osTimeTests os util gtest_main) @@ -71,35 +106,3 @@ add_test( COMMAND osTimeTests ) - -if(TD_LINUX) - -add_executable(osAtomicTests "osAtomicTests.cpp") -target_link_libraries(osAtomicTests os util gtest_main) -add_test( - NAME osAtomicTests - COMMAND osAtomicTests -) - -add_executable(osDirTests "osDirTests.cpp") -target_link_libraries(osDirTests os util gtest_main) -add_test( - NAME osDirTests - COMMAND osDirTests -) - -add_executable(osEnvTests "osEnvTests.cpp") -target_link_libraries(osEnvTests os util gtest_main) -add_test( - NAME osEnvTests - COMMAND osEnvTests -) - -endif() - -add_executable(osSemaphoreTests "osSemaphoreTests.cpp") -target_link_libraries(osSemaphoreTests os util gtest_main) -add_test( - NAME osSemaphoreTests - COMMAND osSemaphoreTests -) diff --git a/source/os/test/osThreadTests.cpp b/source/os/test/osThreadTests.cpp index e7fc4f1356..20964c86bc 100644 --- a/source/os/test/osThreadTests.cpp +++ b/source/os/test/osThreadTests.cpp @@ -29,6 +29,473 @@ #include "os.h" #include "tlog.h" -TEST(osThreadTests, osThreadTests1) { +static int32_t globalVar = 0; +static void funcPtrKey(void *param) { taosMsleep(100); } + +static void *funcPtr200(void *param) { + TdThread thread = taosThreadSelf(); + + TdThreadKey key = {0}; + taosThreadKeyCreate(&key, funcPtrKey); + void *oldVal = taosThreadGetSpecific(key); + taosThreadSetSpecific(key, oldVal); + taosThreadKeyDelete(key); + + int32_t oldType = 0; + taosThreadSetCancelType(-1, &oldType); + taosThreadSetCancelType(0, &oldType); + + int32_t oldState = 0; + taosThreadSetCancelState(-1, &oldState); + taosThreadSetCancelState(0, &oldState); + + int32_t policy; + struct sched_param para; + taosThreadGetSchedParam(thread, &policy, ¶); + taosThreadGetSchedParam(thread, NULL, ¶); + taosThreadGetSchedParam(thread, &policy, NULL); + // taosThreadSetSchedParam(NULL, 0, ¶); + taosThreadSetSchedParam(thread, 0, ¶); + taosMsleep(200); + + return NULL; } + +static void *funcPtr501(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr502(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr503(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtr504(void *param) { + taosMsleep(500); + TdThread thread = taosThreadSelf(); + return NULL; +} + +static void *funcPtrExit1(void *param) { + taosThreadExit(NULL); + return NULL; +} + +static void *funcPtrExit2(void *param) { + taosThreadExit(&globalVar); + return NULL; +} + +TEST(osThreadTests, thread) { + TdThread tid1 = {0}; + TdThread tid2 = {0}; + int32_t reti = 0; + + reti = taosThreadCreate(NULL, NULL, funcPtr200, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCreate(&tid1, NULL, NULL, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCreate(&tid1, NULL, funcPtr200, NULL); + EXPECT_EQ(reti, 0); + taosMsleep(300); + + (void)taosThreadCancel(tid1); + + reti = taosThreadCreate(&tid2, NULL, funcPtr501, NULL); + EXPECT_EQ(reti, 0); + taosMsleep(1000); + (void)taosThreadCancel(tid2); + + taosThreadDetach(tid1); + reti = taosThreadCreate(&tid2, NULL, funcPtr502, NULL); + EXPECT_EQ(reti, 0); + reti = taosThreadDetach(tid2); + + reti = taosThreadEqual(tid1, tid2); + EXPECT_NE(reti, 0); + + reti = taosThreadCreate(&tid2, NULL, funcPtrExit1, NULL); + EXPECT_EQ(reti, 0); + reti = taosThreadCreate(&tid2, NULL, funcPtrExit2, NULL); + EXPECT_EQ(reti, 0); + + taosMsleep(1000); + + // reti = taosThreadCreate(&tid2, NULL, funcPtr503, NULL); + // EXPECT_EQ(reti, 0); + // taosThreadKill(tid2, SIGINT); + + int32_t policy; + struct sched_param para; + taosThreadGetSchedParam(tid2, &policy, ¶); + taosThreadGetSchedParam(tid2, NULL, ¶); + taosThreadGetSchedParam(tid2, &policy, NULL); + // taosThreadSetSchedParam(NULL, 0, ¶); + taosThreadSetSchedParam(tid2, 0, ¶); + + TdThreadKey key = {0}; + taosThreadKeyCreate(&key, funcPtrKey); + void *oldVal = taosThreadGetSpecific(key); + taosThreadSetSpecific(key, oldVal); + taosThreadKeyDelete(key); +} + +TEST(osThreadTests, attr) { + int32_t reti = 0; + TdThreadAttr attr = {0}; + int32_t param = 0; + + reti = taosThreadAttrInit(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrDestroy(NULL); + EXPECT_NE(reti, 0); + + (void)taosThreadAttrInit(&attr); + + reti = taosThreadAttrSetDetachState(&attr, PTHREAD_CREATE_JOINABLE); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetDetachState(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetDetachState(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetDetachState(&attr, ¶m); + EXPECT_EQ(reti, 0); + + reti = taosThreadAttrSetInheritSched(&attr, PTHREAD_INHERIT_SCHED); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetInheritSched(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetInheritSched(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetInheritSched(&attr, ¶m); + EXPECT_EQ(reti, 0); + + struct sched_param schedparam = {0}; + reti = taosThreadAttrGetSchedParam(&attr, &schedparam); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetSchedParam(&attr, &schedparam); + EXPECT_EQ(reti, 0); + schedparam.sched_priority = -1; + reti = taosThreadAttrSetSchedParam(&attr, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetSchedParam(NULL, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedParam(NULL, &schedparam); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedParam(&attr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadAttrSetSchedPolicy(&attr, SCHED_FIFO); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetSchedPolicy(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetSchedPolicy(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetSchedPolicy(&attr, ¶m); + EXPECT_EQ(reti, 0); + + reti = taosThreadAttrSetScope(&attr, PTHREAD_SCOPE_SYSTEM); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetScope(&attr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetScope(NULL, -1); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(NULL, ¶m); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(&attr, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetScope(&attr, ¶m); + EXPECT_EQ(reti, 0); + + size_t stacksize; + reti = taosThreadAttrGetStackSize(&attr, &stacksize); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetStackSize(&attr, stacksize); + EXPECT_EQ(reti, 0); + reti = taosThreadAttrSetStackSize(&attr, 2048); + EXPECT_NE(reti, 0); + reti = taosThreadAttrSetStackSize(NULL, stacksize); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetStackSize(NULL, &stacksize); + EXPECT_NE(reti, 0); + reti = taosThreadAttrGetStackSize(&attr, NULL); + EXPECT_NE(reti, 0); +} + +TEST(osThreadTests, cond) { + int32_t reti = 0; + + reti = taosThreadCondInit(NULL, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondSignal(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondBroadcast(NULL); + EXPECT_NE(reti, 0); + + TdThreadCond cond{0}; + TdThreadMutex mutex = {0}; + reti = taosThreadCondWait(&cond, NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondWait(NULL, &mutex); + EXPECT_NE(reti, 0); + + struct timespec abstime = {0}; + reti = taosThreadCondTimedWait(&cond, NULL, &abstime); + EXPECT_NE(reti, 0); + reti = taosThreadCondTimedWait(NULL, &mutex, &abstime); + EXPECT_NE(reti, 0); + reti = taosThreadCondTimedWait(&cond, &mutex, NULL); + EXPECT_EQ(reti, 0); + + TdThreadCondAttr condattr = {0}; + (void)taosThreadCondAttrInit(&condattr); + reti = taosThreadCondAttrInit(NULL); + EXPECT_NE(reti, 0); + int32_t pshared; + reti = taosThreadCondAttrGetPshared(&condattr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadCondAttrSetPshared(&condattr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadCondAttrSetPshared(&condattr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrGetPshared(&condattr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadCondAttrSetclock(NULL, -1); + EXPECT_NE(reti, 0); + + reti = taosThreadCondAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadCondAttrDestroy(&condattr); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, mutex) { + int32_t reti = 0; + TdThreadMutex mutex; + reti = taosThreadMutexInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadMutexInit(&mutex, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexTryLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexTryLock(&mutex); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexTryLock(&mutex); + EXPECT_NE(reti, 0); + + reti = taosThreadMutexUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexUnlock(&mutex); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexLock(&mutex); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexUnlock(&mutex); + EXPECT_EQ(reti, 0); + + reti = taosThreadMutexDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexDestroy(&mutex); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, mutexAttr) { + int32_t reti = 0; + TdThreadMutexAttr mutexAttr; + reti = taosThreadMutexAttrInit(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrInit(&mutexAttr); + EXPECT_EQ(reti, 0); + + int32_t pshared; + reti = taosThreadMutexAttrGetPshared(&mutexAttr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetPshared(&mutexAttr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetPshared(&mutexAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetPshared(&mutexAttr, NULL); + EXPECT_NE(reti, 0); + + int32_t kind; + reti = taosThreadMutexAttrGetType(&mutexAttr, &kind); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetType(&mutexAttr, kind); + EXPECT_EQ(reti, 0); + reti = taosThreadMutexAttrSetType(&mutexAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrSetType(NULL, kind); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetType(NULL, &kind); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrGetType(&mutexAttr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadMutexAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadMutexAttrDestroy(&mutexAttr); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, rwlock) { + int32_t reti = 0; + TdThreadRwlock rwlock; + reti = taosThreadRwlockInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockInit(&rwlock, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockTryRdlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockTryRdlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockRdlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockRdlock(&rwlock); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockDestroy(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockInit(&rwlock, 0); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockTryWrlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockTryWrlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockWrlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockWrlock(&rwlock); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockUnlock(&rwlock); + EXPECT_EQ(reti, 0); + + reti = taosThreadRwlockDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockDestroy(&rwlock); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, rdlockAttr) { + int32_t reti = 0; + TdThreadRwlockAttr rdlockAttr; + reti = taosThreadRwlockAttrInit(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrInit(&rdlockAttr); + EXPECT_EQ(reti, 0); + + int32_t pshared; + reti = taosThreadRwlockAttrGetPshared(&rdlockAttr, &pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockAttrSetPshared(&rdlockAttr, pshared); + EXPECT_EQ(reti, 0); + reti = taosThreadRwlockAttrSetPshared(&rdlockAttr, -1); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrSetPshared(NULL, pshared); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrGetPshared(NULL, &pshared); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrGetPshared(&rdlockAttr, NULL); + EXPECT_NE(reti, 0); + + reti = taosThreadRwlockAttrDestroy(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadRwlockAttrDestroy(&rdlockAttr); + EXPECT_EQ(reti, 0); +} + +TEST(osThreadTests, spinlock) { + int32_t reti = 0; + + TdThreadSpinlock lock = {0}; + reti = taosThreadSpinInit(&lock, -1); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinLock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinTrylock(&lock); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinDestroy(&lock); + EXPECT_EQ(reti, 0); + + reti = taosThreadSpinInit(&lock, -1); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinTrylock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinUnlock(&lock); + EXPECT_EQ(reti, 0); + reti = taosThreadSpinDestroy(&lock); + EXPECT_EQ(reti, 0); + + reti = taosThreadSpinInit(NULL, 0); + EXPECT_NE(reti, 0); + reti = taosThreadSpinLock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinTrylock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinUnlock(NULL); + EXPECT_NE(reti, 0); + reti = taosThreadSpinDestroy(NULL); + EXPECT_NE(reti, 0); +} + +TEST(osThreadTests, others) { + taosThreadTestCancel(); + taosThreadClear(NULL); +} \ No newline at end of file diff --git a/source/os/test/osTimeTests.cpp b/source/os/test/osTimeTests.cpp index 5c8c837dca..1d34587ad8 100644 --- a/source/os/test/osTimeTests.cpp +++ b/source/os/test/osTimeTests.cpp @@ -33,7 +33,7 @@ TEST(osTimeTests, taosLocalTime) { // Test 1: Test when both timep and result are not NULL time_t timep = 1617531000; // 2021-04-04 18:10:00 struct tm result; - struct tm* local_time = taosLocalTime(&timep, &result, NULL, 0, NULL); + struct tm *local_time = taosLocalTime(&timep, &result, NULL, 0, NULL); ASSERT_NE(local_time, nullptr); ASSERT_EQ(local_time->tm_year, 121); ASSERT_EQ(local_time->tm_mon, 3); @@ -92,4 +92,55 @@ TEST(osTimeTests, taosLocalTime) { local_time = taosLocalTime(&neg_timep3, &result, NULL, 0, NULL); ASSERT_EQ(local_time, nullptr); #endif +} + +TEST(osTimeTests, invalidParameter) { + void *retp = NULL; + int32_t reti = 0; + char buf[1024] = {0}; + char fmt[1024] = {0}; + struct tm tm = {0}; + struct timeval tv = {0}; + + retp = taosStrpTime(buf, fmt, NULL); + EXPECT_EQ(retp, nullptr); + retp = taosStrpTime(NULL, fmt, &tm); + EXPECT_EQ(retp, nullptr); + retp = taosStrpTime(buf, NULL, &tm); + EXPECT_EQ(retp, nullptr); + + reti = taosGetTimeOfDay(NULL); + EXPECT_NE(reti, 0); + + reti = taosTime(NULL); + EXPECT_NE(reti, 0); + + tm.tm_year = 2024; + tm.tm_mon = 10; + tm.tm_mday = 23; + tm.tm_hour = 12; + tm.tm_min = 1; + tm.tm_sec = 0; + tm.tm_isdst = -1; + time_t rett = taosMktime(&tm, NULL); + EXPECT_NE(rett, 0); + + retp = taosLocalTime(NULL, &tm, NULL, 0, NULL); + EXPECT_EQ(retp, nullptr); + + retp = taosLocalTime(&rett, NULL, NULL, 0, NULL); + EXPECT_EQ(retp, nullptr); + + reti = taosSetGlobalTimezone(NULL); + EXPECT_NE(reti, 0); +} + +TEST(osTimeTests, user_mktime64) { + int64_t reti = 0; + + reti = user_mktime64(2024, 10, 23, 12, 3, 2, 1); + EXPECT_NE(reti, 0); + + reti = user_mktime64(2024, 1, 23, 12, 3, 2, 1); + EXPECT_NE(reti, 0); } \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 195cb21618..d2d551a539 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -648,7 +648,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NOT_SINGLE_GROUP, "Not a single-group g TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAGS_NOT_MATCHED, "Tags number not matched") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_TAG_NAME, "Invalid tag name") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG, "Name or password too long") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PASSWD_EMPTY, "Password can not be empty") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY, "Password too short or empty") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PORT, "Port should be an integer that is less than 65535 and greater than 0") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_ENDPOINT, "Endpoint should be in the format of 'fqdn:port'") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_EXPRIE_STATEMENT, "This statement is no longer supported") diff --git a/source/util/test/errorCodeTable.ini b/source/util/test/errorCodeTable.ini index e837954a0b..f67c8ab834 100644 --- a/source/util/test/errorCodeTable.ini +++ b/source/util/test/errorCodeTable.ini @@ -463,7 +463,7 @@ TSDB_CODE_PAR_NOT_SINGLE_GROUP = 0x8000260C TSDB_CODE_PAR_TAGS_NOT_MATCHED = 0x8000260D TSDB_CODE_PAR_INVALID_TAG_NAME = 0x8000260E TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG = 0x80002610 -TSDB_CODE_PAR_PASSWD_EMPTY = 0x80002611 +TSDB_CODE_PAR_PASSWD_TOO_SHORT_OR_EMPTY = 0x80002611 TSDB_CODE_PAR_INVALID_PORT = 0x80002612 TSDB_CODE_PAR_INVALID_ENDPOINT = 0x80002613 TSDB_CODE_PAR_EXPRIE_STATEMENT = 0x80002614 diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 5246d5b31e..c9d28e0623 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1422,34 +1422,36 @@ ,,y,script,./test.sh -f tsim/stream/sliding.sim ,,y,script,./test.sh -f tsim/stream/state0.sim ,,y,script,./test.sh -f tsim/stream/state1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpDelete0.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpDelete1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpDelete2.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpError.sim +,,y,script,./test.sh -f tsim/stream/streamInterpDelete0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpDelete1.sim +,,y,script,./test.sh -f tsim/stream/streamInterpDelete2.sim +,,y,script,./test.sh -f tsim/stream/streamInterpError.sim ,,y,script,./test.sh -f tsim/stream/streamInterpForceWindowClose.sim ,,y,script,./test.sh -f tsim/stream/streamInterpForceWindowClose1.sim ,,y,script,./test.sh -f tsim/stream/streamInterpFwcError.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpHistory.sim +,,y,script,./test.sh -f tsim/stream/streamInterpHistory.sim #,,y,script,./test.sh -f tsim/stream/streamInterpHistory1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpLarge.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpLinear0.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpNext0.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpOther.sim +,,y,script,./test.sh -f tsim/stream/streamInterpLarge.sim +,,y,script,./test.sh -f tsim/stream/streamInterpLinear0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpNext0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpOther.sim #,,y,script,./test.sh -f tsim/stream/streamInterpOther1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpPartitionBy0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPartitionBy0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPartitionBy1.sim #,,y,script,./test.sh -f tsim/stream/streamInterpPrev0.sim #,,y,script,./test.sh -f tsim/stream/streamInterpPrev1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey0.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey2.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey3.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpUpdate.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpUpdate1.sim -#,,y,script,./test.sh -f tsim/stream/streamInterpValue0.sim -#,,y,script,./test.sh -f tsim/stream/streamPrimaryKey0.sim -#,,y,script,./test.sh -f tsim/stream/streamPrimaryKey1.sim -#,,y,script,./test.sh -f tsim/stream/streamPrimaryKey2.sim -#,,y,script,./test.sh -f tsim/stream/streamPrimaryKey3.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey0.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey1.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey2.sim +,,y,script,./test.sh -f tsim/stream/streamInterpPrimaryKey3.sim +,,y,script,./test.sh -f tsim/stream/streamInterpUpdate.sim +,,y,script,./test.sh -f tsim/stream/streamInterpUpdate1.sim +,,y,script,./test.sh -f tsim/stream/streamInterpUpdate2.sim +,,y,script,./test.sh -f tsim/stream/streamInterpValue0.sim +,,y,script,./test.sh -f tsim/stream/streamPrimaryKey0.sim +,,y,script,./test.sh -f tsim/stream/streamPrimaryKey1.sim +,,y,script,./test.sh -f tsim/stream/streamPrimaryKey2.sim +,,y,script,./test.sh -f tsim/stream/streamPrimaryKey3.sim ,,y,script,./test.sh -f tsim/stream/streamTwaError.sim ,,y,script,./test.sh -f tsim/stream/streamTwaFwcFill.sim ,,y,script,./test.sh -f tsim/stream/streamTwaFwcFillPrimaryKey.sim diff --git a/tests/run_local_coverage.sh b/tests/run_local_coverage.sh new file mode 100755 index 0000000000..b2413c4065 --- /dev/null +++ b/tests/run_local_coverage.sh @@ -0,0 +1,388 @@ +#!/bin/bash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function print_color() { + local color="$1" + local message="$2" + echo -e "${color}${message}${NC}" +} + +# Initialization parameter +TDENGINE_DIR="/root/TDinternal/community" +BRANCH="" +TDENGINE_GCDA_DIR="/root/TDinternal/community/debug/" + +# Parse command line parameters +while getopts "hd:b:f:c:u:i:" arg; do + case $arg in + d) + TDENGINE_DIR=$OPTARG + ;; + b) + BRANCH=$OPTARG + ;; + f) + TDENGINE_GCDA_DIR=$OPTARG + ;; + c) + TEST_CASE=$OPTARG + ;; + u) + UNIT_TEST_CASE=$OPTARG + ;; + i) + BRANCH_BUILD=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 0 + ;; + ?) + echo "Usage: ./$(basename $0) -h" + exit 1 + ;; + esac +done + +# Check if the command name is provided +if [ -z "$TDENGINE_DIR" ]; then + echo "Error: TDengine dir is required." + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 1 +fi + + +echo "TDENGINE_DIR = $TDENGINE_DIR" +today=`date +"%Y%m%d"` +TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" + +function pullTDengine() { + print_color "$GREEN" "TDengine pull start" + + # pull parent code + cd "$TDENGINE_DIR/../" + print_color "$GREEN" "git pull parent code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -f + git pull + + # pull tdengine code + cd $TDENGINE_DIR + print_color "$GREEN" "git pull tdengine code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -f + git pull + + print_color "$GREEN" "TDengine pull end" +} + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + print_color "$GREEN" "rebuild.." + rm -rf * + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + make -j 8 install +} + +# Check and get the branch name and build branch +if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "not build,only install!" + cd "$TDENGINE_DIR/../" + git pull + cd "$TDENGINE_DIR/" + git pull + cd $TDENGINE_DIR/debug + make -j 8 install +else + print_color "$GREEN" "Build is not required for this test!" +fi + +function runCasesOneByOne () { + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` + if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then + echo $cmd + case=`echo $cmd | cut -d' ' -f 3` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + + elif [[ "$line" == *"$2"* ]]; then + echo $cmd + if [[ "$cmd" == *"pytest.sh"* ]]; then + cmd=`echo $cmd | cut -d' ' -f 2-20` + fi + case=`echo $cmd | cut -d' ' -f 4-20` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + fi + fi + done < $1 +} + +function runUnitTest() { + print_color "$GREEN" "=== Run unit test case ===" + print_color "$GREEN" " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + print_color "$GREEN" "3.0 unit test done" +} + +function runSimCases() { + print_color "$GREEN" "=== Run sim cases ===" + + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/longtimeruning_cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + +function runPythonCases() { + print_color "$GREEN" "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' longtimeruning_cases.task + + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/longtimeruning_cases.task army + + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task system-test + + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + + +function runTest_all() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + runUnitTest + runSimCases + runPythonCases + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + + +function runTest() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + if [ -n "$TEST_CASE" ] && [ "$TEST_CASE" != "all" ] && [ "$TEST_CASE" != "task" ]; then + TEST_CASE="$TEST_CASE" + print_color "$GREEN" "Test case: $TEST_CASE " + cd $TDENGINE_DIR/tests/script/ && $TEST_CASE + cd $TDENGINE_DIR/tests/army/ && $TEST_CASE + cd $TDENGINE_DIR/tests/system-test/ && $TEST_CASE + cd $TDENGINE_DIR/tests/develop-test/ && $TEST_CASE + elif [ "$TEST_CASE" == "all" ]; then + print_color "$GREEN" "Test case is : parallel_test/longtimeruning_cases.task and all unit cases" + runTest_all + elif [ "$TEST_CASE" == "task" ]; then + print_color "$GREEN" "Test case is only: parallel_test/longtimeruning_cases.task " + runSimCases + runPythonCases + elif [ -n "$UNIT_TEST_CASE" ]; then + UNIT_TEST_CASE="$UNIT_TEST_CASE" + cd $TDENGINE_DIR/debug/build/bin/ && $UNIT_TEST_CASE + else + print_color "$GREEN" "Test case is null" + fi + + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + if [ -n "$TDENGINE_GCDA_DIR" ]; then + TDENGINE_GCDA_DIR="$TDENGINE_GCDA_DIR" + print_color "$GREEN" "Test gcda file dir: $TDENGINE_GCDA_DIR " + else + print_color "$GREEN" "Test gcda file dir is default: /root/TDinternal/community/debug" + fi + + # collect data + lcov -d "$TDENGINE_GCDA_DIR" -capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c' '/*/enterprise/*' '*/docs/*' '*/sim/*'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c' '*/debug/*' '*/tests/*'\ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function stopTaosd { + print_color "$GREEN" "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasod end" +} + +function stopTaosadapter { + print_color "$GREEN" "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasoadapter end" + +} + +WORK_DIR=/root + +date >> $WORK_DIR/date.log +print_color "$GREEN" "Run local coverage test cases" | tee -a $WORK_DIR/date.log + +stopTaosd + +runTest + +lcovFunc + + +date >> $WORK_DIR/date.log +print_color "$GREEN" "End of local coverage test cases" | tee -a $WORK_DIR/date.log + + +# Define coverage information files and output directories +COVERAGE_INFO="$TDENGINE_DIR/coverage.info" +OUTPUT_DIR="$WORK_DIR/coverage_report" + +# Check whether the coverage information file exists +if [ ! -f "$COVERAGE_INFO" ]; then + echo "Error: $COVERAGE_INFO not found!" + exit 1 +fi + +# Generate local HTML reports +genhtml "$COVERAGE_INFO" --branch-coverage --function-coverage --output-directory "$OUTPUT_DIR" + +# Check whether the report was generated successfully +if [ $? -eq 0 ]; then + echo "HTML coverage report generated successfully in $OUTPUT_DIR" + echo "For more details : " + echo "http://192.168.1.61:7000/" +else + echo "Error generating HTML coverage report" + exit 1 +fi + diff --git a/tests/run_local_coverage_only_branch.sh b/tests/run_local_coverage_only_branch.sh new file mode 100755 index 0000000000..9329e448be --- /dev/null +++ b/tests/run_local_coverage_only_branch.sh @@ -0,0 +1,388 @@ +#!/bin/bash + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +function print_color() { + local color="$1" + local message="$2" + echo -e "${color}${message}${NC}" +} + +# Initialization parameter +TDENGINE_DIR="/root/TDinternal/community" +BRANCH="" +TDENGINE_GCDA_DIR="/root/TDinternal/community/debug/" + +# Parse command line parameters +while getopts "hd:b:f:c:u:i:" arg; do + case $arg in + d) + TDENGINE_DIR=$OPTARG + ;; + b) + BRANCH=$OPTARG + ;; + f) + TDENGINE_GCDA_DIR=$OPTARG + ;; + c) + TEST_CASE=$OPTARG + ;; + u) + UNIT_TEST_CASE=$OPTARG + ;; + i) + BRANCH_BUILD=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case]" + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test single case/all cases] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 0 + ;; + ?) + echo "Usage: ./$(basename $0) -h" + exit 1 + ;; + esac +done + +# Check if the command name is provided +if [ -z "$TDENGINE_DIR" ]; then + echo "Error: TDengine dir is required." + echo "Usage: $(basename $0) -d [TDengine dir] -b [Test branch] -i [Build test branch] -f [TDengine gcda dir] -c [Test single case/all cases] -u [Unit test case] " + echo " -d [TDengine dir] [default /root/TDinternal/community; eg: /home/TDinternal/community] " + echo " -b [Test branch] [default local branch; eg:cover/3.0] " + echo " -i [Build test branch] [default no:not build, but still install ;yes:will build and install ] " + echo " -f [TDengine gcda dir] [default /root/TDinternal/community/debug; eg:/root/TDinternal/community/debug/community/source/dnode/vnode/CMakeFiles/vnode.dir/src/tq/] " + echo " -c [Test casingle case/all casesse] [default null; -c all : include parallel_test/longtimeruning_cases.task and all unit cases; -c task : include parallel_test/longtimeruning_cases.task; single case: eg: -c './test.sh -f tsim/stream/streamFwcIntervalFill.sim' ] " + echo " -u [Unit test case] [default null; eg: './schedulerTest' ] " + exit 1 +fi + + +echo "TDENGINE_DIR = $TDENGINE_DIR" +today=`date +"%Y%m%d"` +TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" + +function pullTDengine() { + print_color "$GREEN" "TDengine pull start" + + # pull parent code + cd "$TDENGINE_DIR/../" + print_color "$GREEN" "git pull parent code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -f + git pull + + # pull tdengine code + cd $TDENGINE_DIR + print_color "$GREEN" "git pull tdengine code..." + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -f + git pull + + print_color "$GREEN" "TDengine pull end" +} + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + print_color "$GREEN" "rebuild.." + rm -rf * + makecmd="cmake -DCOVER=true -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + make -j 8 install +} + +# Check and get the branch name and build branch +if [ -n "$BRANCH" ] && [ -z "$BRANCH_BUILD" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "yes" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + pullTDengine + buildTDengine +elif [ -n "$BRANCH_BUILD" ] && [ "$BRANCH_BUILD" == "no" ] ; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "not build,only install!" + cd "$TDENGINE_DIR/../" + git pull + cd "$TDENGINE_DIR/" + git pull + cd $TDENGINE_DIR/debug + make -j 8 install +else + print_color "$GREEN" "Build is not required for this test!" +fi + +function runCasesOneByOne () { + while read -r line; do + if [[ "$line" != "#"* ]]; then + cmd=`echo $line | cut -d',' -f 5` + if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then + echo $cmd + case=`echo $cmd | cut -d' ' -f 3` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + + elif [[ "$line" == *"$2"* ]]; then + echo $cmd + if [[ "$cmd" == *"pytest.sh"* ]]; then + cmd=`echo $cmd | cut -d' ' -f 2-20` + fi + case=`echo $cmd | cut -d' ' -f 4-20` + case_file=`echo $case | tr -d ' /' ` + start_time=`date +%s` + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + end_time=`date +%s` + echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT + fi + fi + done < $1 +} + +function runUnitTest() { + print_color "$GREEN" "=== Run unit test case ===" + print_color "$GREEN" " $TDENGINE_DIR/debug" + cd $TDENGINE_DIR/debug + ctest -j12 + print_color "$GREEN" "3.0 unit test done" +} + +function runSimCases() { + print_color "$GREEN" "=== Run sim cases ===" + + cd $TDENGINE_DIR/tests/script + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/longtimeruning_cases.task sim + + totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + +function runPythonCases() { + print_color "$GREEN" "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' longtimeruning_cases.task + + # army + cd $TDENGINE_DIR/tests/army + runCasesOneByOne ../parallel_test/longtimeruning_cases.task army + + # system-test + cd $TDENGINE_DIR/tests/system-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task system-test + + # develop-test + cd $TDENGINE_DIR/tests/develop-test + runCasesOneByOne ../parallel_test/longtimeruning_cases.task develop-test + + totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalSuccess" -gt "0" ]; then + print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi + + totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` + if [ "$totalFailed" -ne "0" ]; then + print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + fi +} + + +function runTest_all() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + runUnitTest + runSimCases + runPythonCases + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + + +function runTest() { + print_color "$GREEN" "run Test" + + cd $TDENGINE_DIR + [ -d sim ] && rm -rf sim + [ -f $TDENGINE_ALLCI_REPORT ] && rm $TDENGINE_ALLCI_REPORT + + if [ -n "$TEST_CASE" ] && [ "$TEST_CASE" != "all" ] && [ "$TEST_CASE" != "task" ]; then + TEST_CASE="$TEST_CASE" + print_color "$GREEN" "Test case: $TEST_CASE " + cd $TDENGINE_DIR/tests/script/ && $TEST_CASE + cd $TDENGINE_DIR/tests/army/ && $TEST_CASE + cd $TDENGINE_DIR/tests/system-test/ && $TEST_CASE + cd $TDENGINE_DIR/tests/develop-test/ && $TEST_CASE + elif [ "$TEST_CASE" == "all" ]; then + print_color "$GREEN" "Test case is : parallel_test/longtimeruning_cases.task and all unit cases" + runTest_all + elif [ "$TEST_CASE" == "task" ]; then + print_color "$GREEN" "Test case is only: parallel_test/longtimeruning_cases.task " + runSimCases + runPythonCases + elif [ -n "$UNIT_TEST_CASE" ]; then + UNIT_TEST_CASE="$UNIT_TEST_CASE" + cd $TDENGINE_DIR/debug/build/bin/ && $UNIT_TEST_CASE + else + print_color "$GREEN" "Test case is null" + fi + + + stopTaosd + cd $TDENGINE_DIR/tests/script + find . -name '*.sql' | xargs rm -f + + cd $TDENGINE_DIR/tests/pytest + find . -name '*.sql' | xargs rm -f +} + +function lcovFunc { + echo "collect data by lcov" + cd $TDENGINE_DIR + + if [ -n "$TDENGINE_GCDA_DIR" ]; then + TDENGINE_GCDA_DIR="$TDENGINE_GCDA_DIR" + print_color "$GREEN" "Test gcda file dir: $TDENGINE_GCDA_DIR " + else + print_color "$GREEN" "Test gcda file dir is default: /root/TDinternal/community/debug" + fi + + # collect data + lcov -d "$TDENGINE_GCDA_DIR" -capture --rc lcov_branch_coverage=0 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info + + # remove exclude paths + lcov --remove coverage.info \ + '*/contrib/*' '*/test/*' '*/packaging/*' '*/taos-tools/*' '*/taosadapter/*' '*/TSZ/*' \ + '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c' '/*/enterprise/*' '*/docs/*' '*/sim/*'\ + '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\ + '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\ + '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc' '*/branch/*'\ + '*/tthread.c' '*/tversion.c' '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/tencode.h' \ + '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c'\ + '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c' '*/debug/*' '*/tests/*'\ + --rc lcov_branch_coverage=1 -o coverage.info + + # generate result + echo "generate result" + lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT + +} + +function stopTaosd { + print_color "$GREEN" "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasod end" +} + +function stopTaosadapter { + print_color "$GREEN" "Stop taosadapter" + systemctl stop taosadapter.service + PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosadapter + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done + print_color "$GREEN" "Stop tasoadapter end" + +} + +WORK_DIR=/root + +date >> $WORK_DIR/date.log +print_color "$GREEN" "Run local coverage test cases" | tee -a $WORK_DIR/date.log + +stopTaosd + +runTest + +lcovFunc + + +date >> $WORK_DIR/date.log +print_color "$GREEN" "End of local coverage test cases" | tee -a $WORK_DIR/date.log + + +# Define coverage information files and output directories +COVERAGE_INFO="$TDENGINE_DIR/coverage.info" +OUTPUT_DIR="$WORK_DIR/coverage_report" + +# Check whether the coverage information file exists +if [ ! -f "$COVERAGE_INFO" ]; then + echo "Error: $COVERAGE_INFO not found!" + exit 1 +fi + +# Generate local HTML reports +genhtml "$COVERAGE_INFO" --branch-coverage --function-coverage --output-directory "$OUTPUT_DIR" + +# Check whether the report was generated successfully +if [ $? -eq 0 ]; then + echo "HTML coverage report generated successfully in $OUTPUT_DIR" + echo "For more details : " + echo "http://192.168.1.61:7000/" +else + echo "Error generating HTML coverage report" + exit 1 +fi + diff --git a/tests/script/api/test.sh b/tests/script/api/test.sh new file mode 100755 index 0000000000..8b93484fe1 --- /dev/null +++ b/tests/script/api/test.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +make clean + +make + +pgrep taosd || taosd >> /dev/null 2>&1 & + +sleep 10 + +./dbTableRoute localhost +./batchprepare localhost +./stmt-crash localhost +./insertSameTs localhost +./passwdTest localhost +./whiteListTest localhost +./tmqViewTest + diff --git a/tests/script/tsim/stream/streamInterpDelete0.sim b/tests/script/tsim/stream/streamInterpDelete0.sim index 21bac13e4a..440d7ce413 100644 --- a/tests/script/tsim/stream/streamInterpDelete0.sim +++ b/tests/script/tsim/stream/streamInterpDelete0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpDelete1.sim b/tests/script/tsim/stream/streamInterpDelete1.sim index 162da175e8..9413cf8918 100644 --- a/tests/script/tsim/stream/streamInterpDelete1.sim +++ b/tests/script/tsim/stream/streamInterpDelete1.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpDelete2.sim b/tests/script/tsim/stream/streamInterpDelete2.sim index be27dcda49..fb53678eff 100644 --- a/tests/script/tsim/stream/streamInterpDelete2.sim +++ b/tests/script/tsim/stream/streamInterpDelete2.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpError.sim b/tests/script/tsim/stream/streamInterpError.sim index 53a92df772..f0f4e80ade 100644 --- a/tests/script/tsim/stream/streamInterpError.sim +++ b/tests/script/tsim/stream/streamInterpError.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step2 sql create database test2 vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpHistory.sim b/tests/script/tsim/stream/streamInterpHistory.sim index b9685ebf05..9737e7d155 100644 --- a/tests/script/tsim/stream/streamInterpHistory.sim +++ b/tests/script/tsim/stream/streamInterpHistory.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpLarge.sim b/tests/script/tsim/stream/streamInterpLarge.sim index 85203d2d9e..2626f49b6a 100644 --- a/tests/script/tsim/stream/streamInterpLarge.sim +++ b/tests/script/tsim/stream/streamInterpLarge.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpLinear0.sim b/tests/script/tsim/stream/streamInterpLinear0.sim index 7d4b28d545..c52540895b 100644 --- a/tests/script/tsim/stream/streamInterpLinear0.sim +++ b/tests/script/tsim/stream/streamInterpLinear0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpNext0.sim b/tests/script/tsim/stream/streamInterpNext0.sim index abdbeda634..4395031aec 100644 --- a/tests/script/tsim/stream/streamInterpNext0.sim +++ b/tests/script/tsim/stream/streamInterpNext0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpOther.sim b/tests/script/tsim/stream/streamInterpOther.sim index 8553e67ec8..4572bfca56 100644 --- a/tests/script/tsim/stream/streamInterpOther.sim +++ b/tests/script/tsim/stream/streamInterpOther.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 4; diff --git a/tests/script/tsim/stream/streamInterpPartitionBy0.sim b/tests/script/tsim/stream/streamInterpPartitionBy0.sim index 6b222de228..543bb48a1c 100644 --- a/tests/script/tsim/stream/streamInterpPartitionBy0.sim +++ b/tests/script/tsim/stream/streamInterpPartitionBy0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step prev print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpPartitionBy1.sim b/tests/script/tsim/stream/streamInterpPartitionBy1.sim index ecb5e0ee62..c8138ac05f 100644 --- a/tests/script/tsim/stream/streamInterpPartitionBy1.sim +++ b/tests/script/tsim/stream/streamInterpPartitionBy1.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step NULL print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpPrimaryKey0.sim b/tests/script/tsim/stream/streamInterpPrimaryKey0.sim index 9edddff6db..1bbc2a9b5d 100644 --- a/tests/script/tsim/stream/streamInterpPrimaryKey0.sim +++ b/tests/script/tsim/stream/streamInterpPrimaryKey0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpPrimaryKey1.sim b/tests/script/tsim/stream/streamInterpPrimaryKey1.sim index 04a1f299be..0db33c9767 100644 --- a/tests/script/tsim/stream/streamInterpPrimaryKey1.sim +++ b/tests/script/tsim/stream/streamInterpPrimaryKey1.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpPrimaryKey2.sim b/tests/script/tsim/stream/streamInterpPrimaryKey2.sim index f06e1ecd03..0574a1ceec 100644 --- a/tests/script/tsim/stream/streamInterpPrimaryKey2.sim +++ b/tests/script/tsim/stream/streamInterpPrimaryKey2.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpPrimaryKey3.sim b/tests/script/tsim/stream/streamInterpPrimaryKey3.sim index 725cf8d850..23cb0a58e6 100644 --- a/tests/script/tsim/stream/streamInterpPrimaryKey3.sim +++ b/tests/script/tsim/stream/streamInterpPrimaryKey3.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpUpdate.sim b/tests/script/tsim/stream/streamInterpUpdate.sim index 59a188c2a6..394ac1a341 100644 --- a/tests/script/tsim/stream/streamInterpUpdate.sim +++ b/tests/script/tsim/stream/streamInterpUpdate.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpUpdate1.sim b/tests/script/tsim/stream/streamInterpUpdate1.sim index 45f16af35d..3987afa21e 100644 --- a/tests/script/tsim/stream/streamInterpUpdate1.sim +++ b/tests/script/tsim/stream/streamInterpUpdate1.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpUpdate2.sim b/tests/script/tsim/stream/streamInterpUpdate2.sim index 2a71474dd7..cde5b589e8 100644 --- a/tests/script/tsim/stream/streamInterpUpdate2.sim +++ b/tests/script/tsim/stream/streamInterpUpdate2.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/stream/streamInterpValue0.sim b/tests/script/tsim/stream/streamInterpValue0.sim index bce7f0ece6..2cbf61f4bd 100644 --- a/tests/script/tsim/stream/streamInterpValue0.sim +++ b/tests/script/tsim/stream/streamInterpValue0.sim @@ -4,6 +4,8 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +sql alter local 'streamCoverage' '1'; + print step1 print =============== create database sql create database test vgroups 1; diff --git a/tests/script/tsim/tagindex/indexOverflow.sim b/tests/script/tsim/tagindex/indexOverflow.sim index 9e297099d1..99b9023f11 100644 --- a/tests/script/tsim/tagindex/indexOverflow.sim +++ b/tests/script/tsim/tagindex/indexOverflow.sim @@ -76,7 +76,23 @@ while $i < $maxTinyLimit $i = $i + 1 endw +print =============== create database with big tags +$bigTagDb = db +$bigTagStb = stb +$bigTagTb = tb +sql create database $bigTagDb +sql use $bigTagDb + +sql create table $bigTagStb (ts timestamp, f1 int) tags(t1 nchar(100), t2 nchar(800), t3 nchar(800), t4 nchar(800)); + +sql insert into $bigTagTb using $bigTagStb tags("abc", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") values(now, 1); + +$tb = $bigTagTb . 2 +sql insert into $tb using $bigTagStb tags("abc", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") values(now, 1); + +$tb = $bigTagTb . 3 +sql insert into $tb using $bigTagStb tags("abc", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") values(now, 1); -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/password.sim b/tests/script/tsim/user/password.sim index 729097e7e1..7d1eff2f0b 100644 --- a/tests/script/tsim/user/password.sim +++ b/tests/script/tsim/user/password.sim @@ -271,5 +271,16 @@ sql create user u25 pass 'taosdata1~' sql create user u26 pass 'taosdata1,' sql create user u27 pass 'taosdata1.' -return +sql CREATE USER `_xTest1` PASS '2729c41a99b2c5222aa7dd9fc1ce3de7' SYSINFO 1 CREATEDB 0 IS_IMPORT 1 HOST '127.0.0.1'; +sql_error CREATE USER `_xTest2` PASS '2729c41a99b2c5222aa7dd9fc1ce3de7' SYSINFO 1 CREATEDB 0 IS_IMPORT 0 HOST '127.0.0.1'; +sql CREATE USER `_xTest3` PASS '2729c41' SYSINFO 1 CREATEDB 0 IS_IMPORT 1 HOST '127.0.0.1'; +sql_error CREATE USER `_xTest4` PASS '2729c417' SYSINFO 1 CREATEDB 0 IS_IMPORT 0 HOST '127.0.0.1'; +sql CREATE USER `_xTest5` PASS '2xF' SYSINFO 1 CREATEDB 0 IS_IMPORT 1 HOST '127.0.0.1'; +sql_error CREATE USER `_xTest6` PASS '2xF' SYSINFO 1 CREATEDB 0 IS_IMPORT 0 HOST '127.0.0.1'; + + +sql_error alter USER `_xTest1` PASS '2729c41a99b2c5222aa7dd9fc1ce3de7'; +sql_error alter USER `_xTest1` PASS '2729c417'; +sql_error alter USER `_xTest1` PASS '2xF'; + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index 829a8aec27..d3efa61e04 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -4,6 +4,8 @@ import sys import time import os import platform +import random +import string from util.log import * from util.sql import * @@ -12,7 +14,7 @@ from util.dnodes import * import subprocess class TDTestCase: - + updatecfgDict = {'udfdResFuncs': "udf1,udf2"} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") @@ -652,10 +654,20 @@ class TDTestCase: tdDnodes.start(1) time.sleep(2) + def test_udfd_cmd(self): + tdLog.info(" test udfd -V ") + os.system("udfd -V") + tdLog.info(" test udfd -c ") + os.system("udfd -c") + + letters = string.ascii_letters + string.digits + '\\' + path = ''.join(random.choice(letters) for i in range(5000)) - def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + os.system(f"udfd -c {path}") + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring print(" env is ok for all ") + self.test_udfd_cmd() self.prepare_udf_so() self.prepare_data() self.create_udf_function() diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index c99e864e71..f9a3f08bf5 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -11,7 +11,7 @@ from util.dnodes import * import subprocess class TDTestCase: - + updatecfgDict = {'udfdResFuncs': "udf1,udf2"} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") diff --git a/tests/system-test/1-insert/boundary.py b/tests/system-test/1-insert/boundary.py index 25782fd0c3..129b0f275c 100644 --- a/tests/system-test/1-insert/boundary.py +++ b/tests/system-test/1-insert/boundary.py @@ -130,6 +130,8 @@ class TDTestCase: tdSql.error(f'create user {username} pass "test123@#$"') if "Name or password too long" in tdSql.error_info: tdLog.info("error info is true!") + elif "Password too short or empty" in tdSql.error_info: + tdLog.info("error info is true!") else: tdLog.exit("error info is not true") @@ -146,6 +148,10 @@ class TDTestCase: tdSql.error(f'create user {username} pass "{password}@1"') if "Invalid password format" in tdSql.error_info: tdLog.info("error info is true!") + elif "Name or password too long" in tdSql.error_info: + tdLog.info("error info is true!") + elif "Password too short or empty" in tdSql.error_info: + tdLog.info("error info is true!") else: tdLog.exit("error info is not true") def sql_length_check(self): diff --git a/utils/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt index 209982c659..b725ed919a 100644 --- a/utils/tsim/CMakeLists.txt +++ b/utils/tsim/CMakeLists.txt @@ -1,14 +1,29 @@ -aux_source_directory(src TSIM_SRC) -add_executable(tsim ${TSIM_SRC}) -target_link_libraries( - tsim +LIST(APPEND TSIM_SRC src/simEntry.c) +LIST(APPEND TSIM_SRC src/simExec.c) +LIST(APPEND TSIM_SRC src/simParse.c) +LIST(APPEND TSIM_SRC src/simSystem.c) + +ADD_LIBRARY(tsim_static STATIC ${TSIM_SRC}) +TARGET_INCLUDE_DIRECTORIES( + tsim_static + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) +TARGET_LINK_LIBRARIES( + tsim_static PUBLIC ${TAOS_LIB} PUBLIC util PUBLIC common PUBLIC os PUBLIC cjson ) -target_include_directories( + +LIST(APPEND TSIM_EXE_SRC src/simMain.c) +ADD_EXECUTABLE(tsim ${TSIM_EXE_SRC}) +TARGET_LINK_LIBRARIES( tsim - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC tsim_static ) + +IF(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +ENDIF(${BUILD_TEST}) diff --git a/utils/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h index f2360277e0..0ad0bfdea8 100644 --- a/utils/tsim/inc/simInt.h +++ b/utils/tsim/inc/simInt.h @@ -16,6 +16,10 @@ #ifndef _TD_SIM_INT_H_ #define _TD_SIM_INT_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "cJSON.h" @@ -161,8 +165,8 @@ typedef struct _script_t { int32_t type; bool killed; void *taos; - char rows[12]; // number of rows data retrieved - char cols[12]; // number of columns data retrieved + char rows[12]; // number of rows data retrieved + char cols[12]; // number of columns data retrieved char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results char system_exit_code[12]; char system_ret_content[MAX_SYSTEM_RESULT_LEN]; @@ -192,7 +196,7 @@ SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); void *simExecuteScript(void *script); void simInitsimCmdList(); -bool simSystemInit(); +void simSystemInit(); void simSystemCleanUp(); char *simGetVariable(SScript *script, char *varName, int32_t varLen); bool simExecuteExpCmd(SScript *script, char *option); @@ -214,4 +218,11 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *option); bool simExecuteSetBIModeCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); +int32_t simEntry(int32_t argc, char **argv); +void simHandleSignal(int32_t signo, void *sigInfo, void *context); + +#ifdef __cplusplus +} +#endif + #endif /*_TD_SIM_INT_H_*/ diff --git a/utils/tsim/src/simEntry.c b/utils/tsim/src/simEntry.c new file mode 100644 index 0000000000..dd11c21af0 --- /dev/null +++ b/utils/tsim/src/simEntry.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "simInt.h" + +bool simExecSuccess = false; +bool abortExecution = false; +bool useValgrind = false; + +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { + simSystemCleanUp(); + abortExecution = true; +} + +int32_t simEntry(int32_t argc, char **argv) { + char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; + + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { + tstrncpy(configDir, argv[++i], 128); + } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { + tstrncpy(scriptFile, argv[++i], MAX_FILE_NAME_LEN); + } else if (strcmp(argv[i], "-v") == 0) { + useValgrind = true; + } else { + printf("usage: %s [options] \n", argv[0]); + printf(" [-c config]: config directory, default is: %s\n", configDir); + printf(" [-f script]: script filename\n"); + return 0; + } + } + + simInfo("simulator is running ..."); + + simSystemInit(); + taosSetSignal(SIGINT, simHandleSignal); + + SScript *script = simParseScript(scriptFile); + if (script == NULL) { + simError("parse script file:%s failed", scriptFile); + return -1; + } + + simScriptList[++simScriptPos] = script; + simExecuteScript(script); + + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; +} diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExec.c similarity index 100% rename from utils/tsim/src/simExe.c rename to utils/tsim/src/simExec.c index a9772addbb..82ce852b1e 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExec.c @@ -382,8 +382,8 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } -void simReplaceDirSep(char *buf) { #ifdef WINDOWS +void simReplaceDirSep(char *buf) { int i = 0; while (buf[i] != '\0') { if (buf[i] == '/') { @@ -391,8 +391,8 @@ void simReplaceDirSep(char *buf) { } i++; } -#endif } +#endif bool simReplaceStr(char *buf, char *src, char *dst) { bool replaced = false; diff --git a/utils/tsim/src/simMain.c b/utils/tsim/src/simMain.c index cd4a5117b2..0af1e2eec2 100644 --- a/utils/tsim/src/simMain.c +++ b/utils/tsim/src/simMain.c @@ -16,58 +16,7 @@ #define _DEFAULT_SOURCE #include "simInt.h" -bool simExecSuccess = false; -bool abortExecution = false; -bool useValgrind = false; - -void simHandleSignal(int32_t signo, void *sigInfo, void *context) { - simSystemCleanUp(); - abortExecution = true; -} - int32_t main(int32_t argc, char *argv[]) { - char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; - - for (int32_t i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { - tstrncpy(configDir, argv[++i], 128); - } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { - tstrncpy(scriptFile, argv[++i], MAX_FILE_NAME_LEN); - } else if (strcmp(argv[i], "-v") == 0) { - useValgrind = true; - } else { - printf("usage: %s [options] \n", argv[0]); - printf(" [-c config]: config directory, default is: %s\n", configDir); - printf(" [-f script]: script filename\n"); - return 0; - } - } - - if (!simSystemInit()) { - simError("failed to initialize the system"); - simSystemCleanUp(); - return -1; - } - - simInfo("simulator is running ..."); - taosSetSignal(SIGINT, simHandleSignal); - - SScript *script = simParseScript(scriptFile); - if (script == NULL) { - simError("parse script file:%s failed", scriptFile); - return -1; - } - - if (abortExecution) { - simError("execute abort"); - return -1; - } - - simScriptList[++simScriptPos] = script; - simExecuteScript(script); - - int32_t ret = simExecSuccess ? 0 : -1; - simInfo("execute result %d", ret); - - return ret; + // entry function used for unit testing. + return simEntry(argc, argv); } diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index dcf5d6ab12..e1dab0cd6c 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -35,11 +35,10 @@ int32_t simInitCfg() { return 0; } -bool simSystemInit() { +void simSystemInit() { simInitCfg(); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); - return true; } void simSystemCleanUp() {} diff --git a/utils/tsim/test/CMakeLists.txt b/utils/tsim/test/CMakeLists.txt new file mode 100644 index 0000000000..5df85bf903 --- /dev/null +++ b/utils/tsim/test/CMakeLists.txt @@ -0,0 +1,23 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.20) +PROJECT(TDengine) + +FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) +FIND_LIBRARY(LIB_GTEST_STATIC_DIR libgtest.a /usr/lib/ /usr/local/lib /usr/lib64) +FIND_LIBRARY(LIB_GTEST_SHARED_DIR libgtest.so /usr/lib/ /usr/local/lib /usr/lib64) + +IF(HEADER_GTEST_INCLUDE_DIR AND(LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) + MESSAGE(STATUS "gTest library found, build os test") + + INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) + AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) +ENDIF() + +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) + +ADD_EXECUTABLE(simTests "simTests.cpp") +TARGET_LINK_LIBRARIES(simTests os util tsim_static gtest_main) + +ADD_TEST( + NAME simTests + COMMAND simTests +) \ No newline at end of file diff --git a/utils/tsim/test/simTests.cpp b/utils/tsim/test/simTests.cpp new file mode 100644 index 0000000000..e728a1d4cd --- /dev/null +++ b/utils/tsim/test/simTests.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#pragma GCC diagnostic ignored "-Wint-to-pointer-cast" +#pragma GCC diagnostic ignored "-Wpointer-arith" + +#include "simInt.h" + +void simHandleSignal(int32_t signo, void *sigInfo, void *context); + +TEST(simTests, parameters) { + int32_t ret = 0; + int32_t argc = 2; + char *argv[4] = {0}; + + simSystemCleanUp(); + // argv[1] = "-c"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-f"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-v"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // argv[1] = "-h"; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, 0); + + // simHandleSignal(0, NULL, NULL); + + // simDebugFlag = 0; + // argc = 1; + // ret = simEntry(argc, argv); + // EXPECT_EQ(ret, -1); +}