diff --git a/deps/win/x64/dm_static/dmodule.lib b/deps/win/x64/dm_static/dmodule.lib index 36ce1d81d3..55afb81e3f 100644 Binary files a/deps/win/x64/dm_static/dmodule.lib and b/deps/win/x64/dm_static/dmodule.lib differ diff --git a/docs/zh/07-operation/05-monitor.md b/docs/zh/07-operation/05-monitor.md index c5b4cbac06..cd8a9772cd 100644 --- a/docs/zh/07-operation/05-monitor.md +++ b/docs/zh/07-operation/05-monitor.md @@ -8,57 +8,13 @@ toc_max_heading_level: 4 至于如何获取和使用这些监控数据，用户可以使用第三方的监测工具比如 Zabbix 来获取这些保存的系统监测数据，进而将 TDengine 的运行状况无缝集成到现有的 IT 监控系统中。也可以使用 TDengine 提供的 TDinsight 插件，使用该插件用户可以通过 Grafana 平台直观地展示和管理这些监控信息，如下图所示。这为用户提供了灵活的监控选项，以满足不同场景下的运维需求。 -~[通过监控组件管理监控信息](./grafana.png) +![通过监控组件管理监控信息](./grafana.png) ## 配置 taosKeeper 因为 TDengine 的监控数据都通过 taosKeeper 上报并存储，所以本节先介绍 taosKeeper 的配置。 -taosKeeper 的配置文件默认位于 `/etc/taos/taoskeeper.toml`。 下面为一个示例配置文件，更多详细信息见参考手册。其中最为关键的一个配置项是 `database`，它决定了收集到的监控数据存储在目标系统的哪个数据库中。 - -```toml -# gin 框架是否启用 debug -debug = false - -# 服务监听端口, 默认为 6043 -port = 6043 - -# 日志级别，包含 panic、error、info、debug、trace等 -loglevel = "info" - -# 程序中使用协程池的大小 -gopoolsize = 50000 - -# 查询 TDengine 监控数据轮询间隔 -RotationInterval = "15s" - -[tdengine] -host = "127.0.0.1" -port = 6041 -username = "root" -password = "taosdata" - -# 需要被监控的 taosAdapter -[taosAdapter] -address = ["127.0.0.1:6041"] - -[metrics] -# 监控指标前缀 -prefix = "taos" - -# 集群数据的标识符 -cluster = "production" - -# 存放监控数据的数据库 -database = "log" - -# 指定需要监控的普通表 -tables = [] - -# database options for db storing metrics data -[metrics.databaseoptions] -cachemodel = "none" -``` +taosKeeper 的配置文件默认位于 `/etc/taos/taoskeeper.toml`。 详细配置见 [参考手册](../../reference/components/taoskeeper/#配置文件)。其中最为关键的一个配置项是 `database`，它决定了收集到的监控数据存储在目标系统的哪个数据库中。 ## 监控 taosd @@ -66,9 +22,9 @@ cachemodel = "none" 为了简化用户在 TDengine 监控方面的配置工作，TDengine 提供了一个名为 TDinsight 的 Grafana 插件。该插件与 taosKeeper 协同工作，能够实时监控 TDengine 的各项性能指标。 -通过集成 Grafana 和 TDengine 数据源插件，TDinsight 能够读取 taosKeeper 收集并存储的监控数据。这使得用户可以在 Grafana 平台上直观地查看 TDengine 集群的状态、节点信息、读写请求以及资源使用情况等关键指标，实现数据的可视化展示。 +通过集成 Grafana 和 TDengine 数据源插件，TDinsight 能够读取 taosKeeper 收集的监控数据。这使得用户可以在 Grafana 平台上直观地查看 TDengine 集群的状态、节点信息、读写请求以及资源使用情况等关键指标，实现数据的可视化展示。 -此外，TDinsight 还具备针对 vnode、dnode 和 mnode 节点的异常状态告警功能，为开发者提供实时的集群运行状态监控，确保 TDengine 集群的稳定性和可靠性。以下是TDinsight 的详细使用说明，以帮助你充分利用这一强大工具。 +以下是TDinsight 的详细使用说明，以帮助你充分利用这一强大工具。 #### 前置条件 @@ -76,314 +32,31 @@ cachemodel = "none" - TDengine 已安装并正常运行。 - taosAdapter 已经安装并正常运行。 - taosKeeper 已经安装并正常运行。 -- Grafana 已安装并正常运行，以下介绍以 Grafna 10.4.0 为例。 +- Grafana 已安装并正常运行，以下介绍以 Grafna 11.0.0 为例。 同时记录以下信息。 -- taosAdapter 的 RESTful 接口地址，如 http://www.example.com:6041。 +- taosAdapter 的 RESTful 接口地址，如 `http://www.example.com:6041`。 - TDengine 集群的认证信息，包括用户名及密码。 #### 导入仪表盘 TDengine 数据源插件已被提交至 Grafana 官网，完成插件的安装和数据源的创建后，可以进行 TDinsight 仪表盘的导入。 -在 Grafana 的 Home-Dashboards 页面，点击位于右上角的 New → mport 按钮，即可进入 Dashboard 的导入页面，它支持以下两种导入方式。 +在 Grafana 的 ”Home“ -> ”Dashboards“ 页面，点击位于右上角的 ”New“ -> ”import“ 按钮，即可进入 Dashboard 的导入页面，它支持以下两种导入方式。 - Dashboard ID：18180。 - Dashboard URL：https://grafana.com/grafana/dashboards/18180-tdinsight-for-3-x/ -填写以上 Dashboard ID 或 Dashboard URL 以后，点击 Load 按钮，按照向导操作，即可完成导入。导入成功后，Dashboards 列表页面会出现 TDinsight for 3.x 仪盘，点击进入后，就可以看到 TDinsight 中已创建的各个指标的面板，如下图所示： +填写以上 Dashboard ID 或 Dashboard URL 以后，点击 ”Load“ 按钮，按照向导操作，即可完成导入。导入成功后，Dashboards 列表页面会出现 ”TDinsight for 3.x“ 仪表盘，点击进入后，就可以看到 TDinsight 中已创建的各个指标的面板，如下图所示： -![TDinsight 界面示例](./tdinsight.png) +![TDinsight 界面示例](./TDinsight-1-cluster-status.webp) -**注意** 在 TDinsight 界面左上角的 Log from 下拉列表中可以选择 log 数据库。 +**注意** 在 TDinsight 界面左上角的 ”Log from“ 下拉列表中可以选择 `log` 数据库。 -### taosd 监控数据 +### TDengine V3 监控数据 -TDinsight dashboard 数据来源于 log 库（存放监控数据的默认 db，可以在 taoskeeper 配置文件中修改。以下是由 taosd 上报由 taosKeeper 存储在 log 库中的数据。 - -1. taosd\_cluster\_basic 表 - -`taosd_cluster_basic` 表记录集群基础信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|first\_ep|VARCHAR||集群 first ep| -|first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id| -|cluster_version|VARCHAR||tdengine version。例如：3.0.4.0| -|cluster\_id|VARCHAR|TAG|cluster id| - -2. taosd\_cluster\_info 表 - -`taosd_cluster_info` 表记录集群信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|cluster_uptime|DOUBLE||当前 master 节点的uptime。单位：秒| -|dbs\_total|DOUBLE||database 总数| -|tbs\_total|DOUBLE||当前集群 table 总数| -|stbs\_total|DOUBLE||当前集群 stable 总数| -|dnodes\_total|DOUBLE||当前集群 dnode 总数| -|dnodes\_alive|DOUBLE||当前集群 dnode 存活总数| -|mnodes\_total|DOUBLE||当前集群 mnode 总数| -|mnodes\_alive|DOUBLE||当前集群 mnode 存活总数| -|vgroups\_total|DOUBLE||当前集群 vgroup 总数| -|vgroups\_alive|DOUBLE||当前集群 vgroup 存活总数| -|vnodes\_total|DOUBLE||当前集群 vnode 总数| -|vnodes\_alive|DOUBLE||当前集群 vnode 存活总数| -|connections\_total|DOUBLE||当前集群连接总数| -|topics\_total|DOUBLE||当前集群 topic 总数| -|streams\_total|DOUBLE||当前集群 stream 总数| -|grants_expire\_time|DOUBLE||认证过期时间，企业版有效，社区版为 DOUBLE 最大值| -|grants_timeseries\_used|DOUBLE||已用测点数| -|grants_timeseries\_total|DOUBLE||总测点数，开源版本为 DOUBLE 最大值| -|cluster\_id|VARCHAR|TAG|cluster id| - -3. taosd\_vgroups\_info 表 - -`taosd_vgroups_info` 表记录虚拟节点组信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|tables\_num|DOUBLE||vgroup 中 table 数量| -|status|DOUBLE||vgroup 状态, 取值范围：unsynced = 0, ready = 1| -|vgroup\_id|VARCHAR|TAG|vgroup id| -|database\_name|VARCHAR|TAG|vgroup 所属的 database 名字| -|cluster\_id|VARCHAR|TAG|cluster id| - -4. taosd\_dnodes\_info 表 - -`taosd_dnodes_info` 记录 dnode 信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|uptime|DOUBLE||dnode uptime，单位：秒| -|cpu\_engine|DOUBLE||taosd cpu 使用率，从 `/proc//stat` 读取| -|cpu\_system|DOUBLE||服务器 cpu 使用率，从 `/proc/stat` 读取| -|cpu\_cores|DOUBLE||服务器 cpu 核数| -|mem\_engine|DOUBLE||taosd 内存使用率，从 `/proc//status` 读取| -|mem\_free|DOUBLE||服务器可用内存，单位 KB| -|mem\_total|DOUBLE||服务器内存总量，单位 KB| -|disk\_used|DOUBLE||data dir 挂载的磁盘使用量，单位 bytes| -|disk\_total|DOUBLE||data dir 挂载的磁盘总容量，单位 bytes| -|system\_net\_in|DOUBLE||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 byte/s| -|system\_net\_out|DOUBLE||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 byte/s| -|io\_read|DOUBLE||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 byte/s| -|io\_write|DOUBLE||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 byte/s| -|io\_read\_disk|DOUBLE||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 byte/s| -|io\_write\_disk|DOUBLE||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 byte/s| -|vnodes\_num|DOUBLE||dnode 上 vnodes 数量| -|masters|DOUBLE||dnode 上 master node 数量| -|has\_mnode|DOUBLE||dnode 是否包含 mnode，取值范围：包含=1,不包含=0| -|has\_qnode|DOUBLE||dnode 是否包含 qnode，取值范围：包含=1,不包含=0| -|has\_snode|DOUBLE||dnode 是否包含 snode，取值范围：包含=1,不包含=0| -|has\_bnode|DOUBLE||dnode 是否包含 bnode，取值范围：包含=1,不包含=0| -|error\_log\_count|DOUBLE||error 总数| -|info\_log\_count|DOUBLE||info 总数| -|debug\_log\_count|DOUBLE||debug 总数| -|trace\_log\_count|DOUBLE||trace 总数| -|dnode\_id|VARCHAR|TAG|dnode id| -|dnode\_ep|VARCHAR|TAG|dnode endpoint| -|cluster\_id|VARCHAR|TAG|cluster id| - -5. taosd\_dnodes\_status 表 - -`taosd_dnodes_status` 表记录 dnode 状态信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|status|DOUBLE||dnode 状态,取值范围：ready=1，offline =0| -|dnode\_id|VARCHAR|TAG|dnode id| -|dnode\_ep|VARCHAR|TAG|dnode endpoint| -|cluster\_id|VARCHAR|TAG|cluster id| - -6. taosd\_dnodes\_log\_dir 表 - -`taosd_dnodes_log_dir` 表记录 log 目录信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|avail|DOUBLE||log 目录可用空间。单位 byte| -|used|DOUBLE||log 目录已使用空间。单位 byte| -|total|DOUBLE||log 目录空间。单位 byte| -|name|VARCHAR|TAG|log 目录名，一般为 `/var/log/taos/`| -|dnode\_id|VARCHAR|TAG|dnode id| -|dnode\_ep|VARCHAR|TAG|dnode endpoint| -|cluster\_id|VARCHAR|TAG|cluster id| - -7. taosd\_dnodes\_data\_dir 表 - -`taosd_dnodes_data_dir` 表记录 data 目录信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|avail|DOUBLE||data 目录可用空间。单位 byte| -|used|DOUBLE||data 目录已使用空间。单位 byte| -|total|DOUBLE||data 目录空间。单位 byte| -|level|VARCHAR|TAG|0、1、2 多级存储级别| -|name|VARCHAR|TAG|data 目录，一般为 `/var/lib/taos`| -|dnode\_id|VARCHAR|TAG|dnode id| -|dnode\_ep|VARCHAR|TAG|dnode endpoint| -|cluster\_id|VARCHAR|TAG|cluster id| - -8. taosd\_mnodes\_info 表 - -`taosd_mnodes_info` 表记录 mnode 角色信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|role|DOUBLE||mnode 角色， 取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104| -|mnode\_id|VARCHAR|TAG|master node id| -|mnode\_ep|VARCHAR|TAG|master node endpoint| -|cluster\_id|VARCHAR|TAG|cluster id| - -9. taosd\_vnodes\_role 表 - -`taosd_vnodes_role` 表记录虚拟节点角色信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|vnode\_role|DOUBLE||vnode 角色，取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104| -|vgroup\_id|VARCHAR|TAG|dnode id| -|dnode\_id|VARCHAR|TAG|dnode id| -|database\_name|VARCHAR|TAG|vgroup 所属的 database 名字| -|cluster\_id|VARCHAR|TAG|cluster id| - -10. taosd\_sql\_req 表 - -`taosd_sql_req` 记录授权信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|count|DOUBLE||sql 数量| -|result|VARCHAR|TAG|sql的执行结果，取值范围：Success, Failed| -|username|VARCHAR|TAG|执行sql的user name| -|sql\_type|VARCHAR|TAG|sql类型，取值范围：inserted_rows| -|dnode\_id|VARCHAR|TAG|dnode id| -|dnode\_ep|VARCHAR|TAG|dnode endpoint| -|vgroup\_id|VARCHAR|TAG|dnode id| -|cluster\_id|VARCHAR|TAG|cluster id| - -11. taos\_sql\_req 表 - -`taos_sql_req` 记录授权信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|count|DOUBLE||sql 数量| -|result|VARCHAR|TAG|sql的执行结果，取值范围：Success, Failed| -|username|VARCHAR|TAG|执行sql的user name| -|sql\_type|VARCHAR|TAG|sql类型，取值范围：select, insert，delete| -|cluster\_id|VARCHAR|TAG|cluster id| - -12. taos\_slow\_sql 表 - -`taos_slow_sql` 记录授权信息。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|count|DOUBLE||sql 数量| -|result|VARCHAR|TAG|sql的执行结果，取值范围：Success, Failed| -|username|VARCHAR|TAG|执行sql的user name| -|duration|VARCHAR|TAG|sql执行耗时，取值范围：3-10s,10-100s,100-1000s,1000s-| -|cluster\_id|VARCHAR|TAG|cluster id| - -13. keeper\_monitor 表 - -`keeper_monitor` 记录 taoskeeper 监控数据。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|ts|TIMESTAMP||timestamp| -|cpu|DOUBLE||cpu 使用率| -|mem|DOUBLE||内存使用率| -|identify|NCHAR|TAG|| - -14. taosadapter\_restful\_http\_request\_total 表 - -`taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|gauge|DOUBLE||监控指标值| -|client\_ip|NCHAR|TAG|client ip| -|endpoint|NCHAR|TAG|taosadpater endpoint| -|request\_method|NCHAR|TAG|request method| -|request\_uri|NCHAR|TAG|request uri| -|status\_code|NCHAR|TAG|status code| - -15. taosadapter\_restful\_http\_request\_fail 表 - -`taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|gauge|DOUBLE||监控指标值| -|client\_ip|NCHAR|TAG|client ip| -|endpoint|NCHAR|TAG|taosadpater endpoint| -|request\_method|NCHAR|TAG|request method| -|request\_uri|NCHAR|TAG|request uri| -|status\_code|NCHAR|TAG|status code| - -16. taosadapter\_restful\_http\_request\_in\_flight 表 - -`taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|gauge|DOUBLE||监控指标值| -|endpoint|NCHAR|TAG|taosadpater endpoint| - -17. taosadapter\_restful\_http\_request\_summary\_milliseconds 表 - -`taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|count|DOUBLE||| -|sum|DOUBLE||| -|0.5|DOUBLE||| -|0.9|DOUBLE||| -|0.99|DOUBLE||| -|0.1|DOUBLE||| -|0.2|DOUBLE||| -|endpoint|NCHAR|TAG|taosadpater endpoint| -|request\_method|NCHAR|TAG|request method| -|request\_uri|NCHAR|TAG|request uri| - -18. taosadapter\_system\_mem\_percent 表 - -`taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|gauge|DOUBLE||监控指标值| -|endpoint|NCHAR|TAG|taosadpater endpoint| - -19. taosadapter\_system\_cpu\_percent 表 - -`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 - -|field|type|is\_tag|comment| -|:----|:---|:-----|:------| -|\_ts|TIMESTAMP||timestamp| -|gauge|DOUBLE||监控指标值| -|endpoint|NCHAR|TAG|taosadpater endpoint| +TDinsight dashboard 数据来源于 `log` 库（存放监控数据的默认数据库，可以在 taoskeeper 配置文件中修改）。”TDinsight for 3.x“ 仪表盘查询了 taosd 和 TaosAdapter 的监控指标。 +- taosd 的监控指标请参考 [taosd 监控指标](../../reference/components/taosd/#taosd-监控指标) +- taosAdapter 的监控指标请参考 [taosAdapter 监控指标](../../reference/components/taosadapter/#taosadapter-监控指标) ## 监控 taosX @@ -396,13 +69,13 @@ taosX 是 TDengine 中提供零代码数据接入能力的核心组件，对它 ### 版本支持 1. TDengine 企业版本 3.2.3.0 或以上版本包含的 taosX 才包含此功能。如果单独安装 taosX，需要 taosX 1.5.0 或以上版本。 -1. 需要安装 Grafana 插件 [TDengie Datasource v3.5.0](https://grafana.com/grafana/plugins/tdengine-datasource/) 或以上版本。 +2. 需要安装 Grafana 插件 [TDengie Datasource v3.5.0](https://grafana.com/grafana/plugins/tdengine-datasource/) 或以上版本。 ### 准备工作 -假设你已经部署好了 taosd，taosAdapter 和 taosAdapter。 那么还需要： -2. 启动 taosX 服务。 -3. 部署 Grafana ，安装 TDengine Datasource 插件，配置好数据源。 +假设你已经部署好了 taosd，taosAdapter 和 taosAdapter。 那么还需要： +1. 启动 taosX 服务。 +2. 部署 Grafana ，安装 TDengine Datasource 插件，配置好数据源。 ### 配置 taosX @@ -426,52 +99,50 @@ toasX 的配置文件(默认 /etc/taos/taosx.toml) 中与 monitor 相关的配 | port | --monitor-port | MONITOR_PORT | taosKeeper 服务的端口 | | 6043 | | interval | --monitor-interval | MONITTOR_INTERVAL | taosX 发送 metrics 数据到 taosKeeper 的时间间隔，单位秒 | 1-10 | 10 | -TDinsight for taosX - -"TDinsight for taosX" 专门为 taosX 监控创建的 Grafana 面板。使用前需要先导入这个面板。 - ### 基于 TDinsight 监控 tasoX +"TDinsight for taosX" 是专门为 taosX 监控创建的 Grafana 面板。使用前需要先导入这个面板。 + #### 进入面板 -1. 选择 TDengine Datasource - ![TDengine Datasource](./pic/monitor-01.jpg) -2. 点击 “Dashboard”, 选择 TDinsight for taosX 面板。（第一次使用需要先导入）。 - ![Dashboard](./pic/monitor-02.jpg) +1. 在 Grafana 界面菜单中点击 ”Data sources“， 然后选择已经配置好的 TDengine 数据源。 +2. 在数据源配置界面选择 “Dashboard” Tab, 然后导入 ”TDinsight for taosX“ 面板（第一次使用需要先导入）。 下面是一个示例图： + ![monitor rows](./pic/monitor-04.jpg) - 该面板每一行代表一个或一类监控对象。最上面是 taosX 监控行，然后是 Agent 监控行, 最后是各类数据写入任务的监控。 - :::note - 1. 如果打开这个面板后看不到任何数据，你很可能需要点击左上角的数据库列表（即 “Log from” 下拉菜单），切换到监控数据所在的数据库。 - 2. 数据库包含多少个 Agent 的数据就会自动创建多少个 Agent 行。(如上图) - - ::: + + 该面板每一行代表一个或一类监控对象。最上面是 taosX 监控行，然后是 Agent 监控行, 最后是各类数据写入任务的监控。 + :::note + - 如果打开这个面板后看不到任何数据，你很可能需要点击左上角的数据库列表（即 “Log from” 下拉菜单），切换到监控数据所在的数据库。 + - 数据库包含多少个 Agent 的数据就会自动创建多少个 Agent 行。(如上图) + ::: #### 监控示例 -1. taosX 监控示例 +1. taosX 监控示例图 -![monitor taosx](./pic/monitor-03.png) + ![monitor taosx](./pic/monitor-03.png) -2. Agent 监控示例 +2. Agent 监控示例图 -![monitor agent](./pic/monitor-09.jpg) + ![monitor agent](./pic/monitor-09.jpg) -3. TDengine2 数据源监控示例 +3. TDengine2 数据源监控示例图 -![monitor tdengine2](./pic/monitor-05.png) + ![monitor tdengine2](./pic/monitor-05.png) -:::info -监控面板只展示了数据写入任务的部分监控指标，在 Explorer 页面上有更全面的监控指标，且有每个指标的具体说明。 + :::info + 监控面板只展示了数据写入任务的部分监控指标，在 Explorer 页面上有更全面的监控指标，且有每个指标的具体说明。 -::: + ::: -3. TDengine3 数据源监控示例 -![monitor tdengine3](./pic/monitor-06.jpg) +4. TDengine3 数据源监控示例图 + + ![monitor tdengine3](./pic/monitor-06.jpg) -4. 其它数据源监控示例 -![monitor task](./pic/monitor-10.jpg) +5. 其它数据源监控示例图 + ![monitor task](./pic/monitor-10.jpg) #### 限制 diff --git a/docs/zh/07-operation/TDinsight-1-cluster-status.webp b/docs/zh/07-operation/TDinsight-1-cluster-status.webp new file mode 100644 index 0000000000..7f7792b2e9 Binary files /dev/null and b/docs/zh/07-operation/TDinsight-1-cluster-status.webp differ diff --git a/docs/zh/07-operation/tdinsight.png b/docs/zh/07-operation/tdinsight.png deleted file mode 100644 index 0d90950318..0000000000 Binary files a/docs/zh/07-operation/tdinsight.png and /dev/null differ diff --git a/docs/zh/08-develop/09-udf.md b/docs/zh/08-develop/09-udf.md index 45e4ae6134..700bbb2ae0 100644 --- a/docs/zh/08-develop/09-udf.md +++ b/docs/zh/08-develop/09-udf.md @@ -6,29 +6,28 @@ toc_max_heading_level: 4 ## UDF 简介 -在某些应用场景中，应用逻辑需要的查询功能无法直接使用TDengine内置的函数来实现。TDengine允许编写用户自定义函数（UDF），以便解决特殊应用场景中的使用需求。UDF在集群中注册成功后，可以像系统内置函数一样在SQL中调用，就使用角度而言没有任何区别。UDF分为标量函数和聚合函数。标量函数对每行数据输出一个值，如求绝对值abs、正弦函数sin、字符串拼接函数concat等。聚合函数对多行数据输出一个值，如求平均数avg、取最大值max等。 +在某些应用场景中，应用逻辑需要的查询功能无法直接使用内置函数来实现，TDengine 允许编写用户自定义函数（UDF），以便解决特殊应用场景中的使用需求。UDF 在集群中注册成功后，可以像系统内置函数一样在 SQL 中调用，就使用角度而言没有任何区别。UDF 分为标量函数和聚合函数。标量函数对每行数据输出一个值，如求绝对值（abs）、正弦函数（sin）、字符串拼接函数（concat）等。聚合函数对多行数据输出一个值，如求平均数（avg）、取最大值（max）等。 -TDengine支持用C和Python两种编程语言编写UDF。C语言编写的UDF与内置函数的性能几乎相同，Python语言编写的UDF可以利用丰富的Python运算库。为了避免UDF执行中发生异常影响数据库服务，TDengine使用了进程分离技术，把UDF的执行放到另一个进程中完成，即使用户编写的UDF崩溃，也不会影响TDengine的正常运行。 +TDengine 支持用 C 和 Python 两种编程语言编写 UDF。C 语言编写的 UDF 与内置函数的性能几乎相同，Python 语言编写的 UDF 可以利用丰富的 Python 运算库。为了避免 UDF 执行中发生异常影响数据库服务，TDengine 使用了进程分离技术，把 UDF 的执行放到另一个进程中完成，即使用户编写的 UDF 崩溃，也不会影响 TDengine 的正常运行。 ## 用 C 语言开发 UDF 使用 C 语言实现 UDF 时，需要实现规定的接口函数 - 标量函数需要实现标量接口函数 scalarfn 。 -- 聚合函数需要实现聚合接口函数 aggfn_start ， aggfn ， aggfn_finish。 -- 如果需要初始化，实现 udf_init；如果需要清理工作，实现udf_destroy。 - -接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（`_start`, `_finish`, `_init`, `_destroy`)的连接。列表中的scalarfn，aggfn, udf需要替换成udf函数名。 +- 聚合函数需要实现聚合接口函数 aggfn_start、aggfn、aggfn_finish。 +- 如果需要初始化，实现 udf_init。 +- 如果需要清理工作，实现 udf_destroy。 ### 接口定义 -在TDengine中，UDF的接口函数名称可以是UDF名称，也可以是UDF名称和特定后缀（如_start、_finish、_init、_destroy）的连接。后面内容中描述的函数名称，例如scalarfn、aggfn，需要替换成UDF名称。。 +接口函数的名称是 UDF 名称，或者是 UDF 名称和特定后缀（_start、_finish、_init、_destroy）的连接。后面内容中描述的函数名称，例如 scalarfn、aggfn，需要替换成 UDF 名称。 #### 标量函数接口 标量函数是一种将输入数据转换为输出数据的函数，通常用于对单个数据值进行计算和转换。标量函数的接口函数原型如下。 ```c -int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn) +int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn); ``` 主要参数说明如下。 - inputDataBlock：输入的数据块。 @@ -37,23 +36,22 @@ int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn *resultColumn) #### 聚合函数接口 聚合函数是一种特殊的函数，用于对数据进行分组和计算，从而生成汇总信息。聚合函数的工作原理如下。 -- 初始化结果缓冲区：首先调用aggfn_start函数，生成一个结果缓冲区（result buffer），用于存储中间结果。 +- 初始化结果缓冲区：首先调用 aggfn_start 函数，生成一个结果缓冲区（result buffer），用于存储中间结果。 - 分组数据：相关数据会被分为多个行数据块（row data block），每个行数据块包含一组具有相同分组键（grouping key）的数据。 -- 更新中间结果：对于每个数据块，调用aggfn函数更新中间结果。aggfn函数会根据聚合函数的类型（如sum、avg、count等）对数据进行相应的计算，并将计算结 +- 更新中间结果：对于每个数据块，调用 aggfn 函数更新中间结果。aggfn 函数会根据聚合函数的类型（如 sum、avg、count 等）对数据进行相应的计算，并将计算结 果存储在结果缓冲区中。 -- 生成最终结果：在所有数据块的中间结果更新完成后，调用aggfn_finish函数从结果缓冲区中提取最终结果。最终结果通常只包含0条或1条数据，具体取决于聚 +- 生成最终结果：在所有数据块的中间结果更新完成后，调用 aggfn_finish 函数从结果缓冲区中提取最终结果。最终结果只包含 0 条或 1 条数据，具体取决于聚 合函数的类型和输入数据。 聚合函数的接口函数原型如下。 ```c -int32_t aggfn_start(SUdfInterBuf *interBuf) -int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) -int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result) +int32_t aggfn_start(SUdfInterBuf *interBuf); +int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf); +int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result); ``` - -其中 aggfn 是函数名的占位符。首先调用aggfn_start生成结果buffer，然后相关的数据会被分为多个行数据块，对每个数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 +其中 aggfn 是函数名的占位符。首先调用 aggfn_start 生成结果 buffer，然后相关的数据会被分为多个行数据块，对每个数据块调用 aggfn 用数据块更新中间结果，最后再调用 aggfn_finish 从中间结果产生最终结果，最终结果只能含 0 或 1 条结果数据。 主要参数说明如下。 - interBuf：中间结果缓存区。 @@ -61,29 +59,49 @@ int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result) - newInterBuf：新的中间结果缓冲区。 - result：最终结果。 - #### 初始化和销毁接口 -初始化和销毁接口是标量函数和聚合函数共同使用的接口，相关API如下。 +初始化和销毁接口是标量函数和聚合函数共同使用的接口，相关 API 如下。 ```c int32_t udf_init() int32_t udf_destroy() ``` -其中，udf_init函数完成初始化工作，udf_destroy函数完成清理工作。如果没有初始化工作，无须定义udf_init函数；如果没有清理工作，无须定义udf_destroy函数。 +其中，udf_init 函数完成初始化工作，udf_destroy 函数完成清理工作。如果没有初始化工作，无须定义 udf_init 函数；如果没有清理工作，无须定义 udf_destroy 函数。 ### 标量函数模板 -用C语言开发标量函数的模板如下。 +用 C 语言开发标量函数的模板如下。 ```c +#include "taos.h" +#include "taoserror.h" +#include "taosudf.h" + +// Initialization function. +// If no initialization, we can skip definition of it. +// The initialization function shall be concatenation of the udf name and _init suffix. +// @return error number defined in taoserror.h int32_t scalarfn_init() { + // initialization. return TSDB_CODE_SUCCESS; } + +// Scalar function main computation function. +// @param inputDataBlock, input data block composed of multiple columns with each column defined by SUdfColumn +// @param resultColumn, output column +// @return error number defined in taoserror.h int32_t scalarfn(SUdfDataBlock* inputDataBlock, SUdfColumn* resultColumn) { + // read data from inputDataBlock and process, then output to resultColumn. return TSDB_CODE_SUCCESS; } + +// Cleanup function. +// If no cleanup related processing, we can skip definition of it. +// The destroy function shall be concatenation of the udf name and _destroy suffix. +// @return error number defined in taoserror.h int32_t scalarfn_destroy() { + // clean up return TSDB_CODE_SUCCESS; } ``` @@ -91,53 +109,211 @@ int32_t scalarfn_destroy() { 用C语言开发聚合函数的模板如下。 ```c +#include "taos.h" +#include "taoserror.h" +#include "taosudf.h" + +// Initialization function. +// If no initialization, we can skip definition of it. +// The initialization function shall be concatenation of the udf name and _init suffix. +// @return error number defined in taoserror.h int32_t aggfn_init() { + // initialization. return TSDB_CODE_SUCCESS; } + +// Aggregate start function. +// The intermediate value or the state(@interBuf) is initialized in this function. +// The function name shall be concatenation of udf name and _start suffix. +// @param interbuf intermediate value to initialize +// @return error number defined in taoserror.h int32_t aggfn_start(SUdfInterBuf* interBuf) { + // initialize intermediate value in interBuf return TSDB_CODE_SUCCESS; } + +// Aggregate reduce function. +// This function aggregate old state(@interbuf) and one data bock(inputBlock) and output a new state(@newInterBuf). +// @param inputBlock input data block +// @param interBuf old state +// @param newInterBuf new state +// @return error number defined in taoserror.h int32_t aggfn(SUdfDataBlock* inputBlock, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + // read from inputBlock and interBuf and output to newInterBuf return TSDB_CODE_SUCCESS; } + +// Aggregate function finish function. +// This function transforms the intermediate value(@interBuf) into the final output(@result). +// The function name must be concatenation of aggfn and _finish suffix. +// @interBuf : intermediate value +// @result: final result +// @return error number defined in taoserror.h int32_t int32_t aggfn_finish(SUdfInterBuf* interBuf, SUdfInterBuf *result) { + // read data from inputDataBlock and process, then output to result return TSDB_CODE_SUCCESS; } + +// Cleanup function. +// If no cleanup related processing, we can skip definition of it. +// The destroy function shall be concatenation of the udf name and _destroy suffix. +// @return error number defined in taoserror.h int32_t aggfn_destroy() { + // clean up return TSDB_CODE_SUCCESS; } ``` ### 编译 -在TDengine中，为了实现UDF，需要编写C语言源代码，并按照TDengine的规范编译为动态链接库文件。 -按照前面描述的规则，准备UDF的源代码bit_and.c。以Linux操作系统为例，执行如下指令，编译得到动态链接库文件。 +在 TDengine 中，为了实现 UDF，需要编写 C 语言源代码，并按照 TDengine 的规范编译为动态链接库文件。 +按照前面描述的规则，准备 UDF 的源代码 bit_and.c。以 Linux 操作系统为例，执行如下指令，编译得到动态链接库文件。 ```shell -gcc-g-O0-fPIC-sharedbit_and.c-olibbitand.so +gcc -g -O0 -fPIC -shared bit_and.c -o libbitand.so ``` -为了保证可靠运行，推荐使用7.5及以上版本的GCC。 +为了保证可靠运行，推荐使用 7.5 及以上版本的 GCC。 + +### C UDF 数据结构 +```c +typedef struct SUdfColumnMeta { + int16_t type; + int32_t bytes; + uint8_t precision; + uint8_t scale; +} SUdfColumnMeta; + +typedef struct SUdfColumnData { + int32_t numOfRows; + int32_t rowsAlloc; + union { + struct { + int32_t nullBitmapLen; + char *nullBitmap; + int32_t dataLen; + char *data; + } fixLenCol; + + struct { + int32_t varOffsetsLen; + int32_t *varOffsets; + int32_t payloadLen; + char *payload; + int32_t payloadAllocLen; + } varLenCol; + }; +} SUdfColumnData; + +typedef struct SUdfColumn { + SUdfColumnMeta colMeta; + bool hasNull; + SUdfColumnData colData; +} SUdfColumn; + +typedef struct SUdfDataBlock { + int32_t numOfRows; + int32_t numOfCols; + SUdfColumn **udfCols; +} SUdfDataBlock; + +typedef struct SUdfInterBuf { + int32_t bufLen; + char *buf; + int8_t numOfResult; //zero or one +} SUdfInterBuf; +``` +数据结构说明如下： + +- SUdfDataBlock 数据块包含行数 numOfRows 和列数 numCols。udfCols[i] (0 \<= i \<= numCols-1)表示每一列数据，类型为SUdfColumn*。 +- SUdfColumn 包含列的数据类型定义 colMeta 和列的数据 colData。 +- SUdfColumnMeta 成员定义同 taos.h 数据类型定义。 +- SUdfColumnData 数据可以变长，varLenCol 定义变长数据，fixLenCol 定义定长数据。 +- SUdfInterBuf 定义中间结构 buffer，以及 buffer 中结果个数 numOfResult + +为了更好的操作以上数据结构，提供了一些便利函数，定义在 taosudf.h。 + + +### C UDF 示例代码 + +#### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/bit_and.c) + +bit_add 实现多列的按位与功能。如果只有一列，返回这一列。bit_add 忽略空值。 + +
+bit_and.c + +```c +{{#include tests/script/sh/bit_and.c}} +``` + +
+ +#### 聚合函数示例1 返回值为数值类型 [l2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/l2norm.c) + +l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 + +
+l2norm.c + +```c +{{#include tests/script/sh/l2norm.c}} +``` + +
+ +#### 聚合函数示例2 返回值为字符串类型 [max_vol](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/max_vol.c) + +max_vol 实现了从多个输入的电压列中找到最大电压，返回由设备 ID + 最大电压所在（行，列）+ 最大电压值 组成的组合字符串值 + +创建表： +```bash +create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId varchar(16)); +``` +创建自定义函数： +```bash +create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; +``` +使用自定义函数： +```bash +select max_vol(vol1, vol2, vol3, deviceid) from battery; +``` + +
+max_vol.c + +```c +{{#include tests/script/sh/max_vol.c}} +``` + +
## 用 Python 语言开发 UDF ### 准备环境 准备环境的具体步骤如下： -- 第1步，准备好Python运行环境。 -- 第2步，安装Python包taospyudf。命令如下。 +- 第1步，准备好 Python 运行环境。 +- 第2步，安装 Python 包 taospyudf。命令如下。 ```shell pip3 install taospyudf ``` -- 第3步，执行命令ldconfig。 -- 第4步，启动taosd服务。 +- 第3步，执行命令 ldconfig。 +- 第4步，启动 taosd 服务。 + +安装过程中会编译 C++ 源码，因此系统上要有 cmake 和 gcc。编译生成的 libtaospyudf.so 文件自动会被复制到 /usr/local/lib/ 目录，因此如果是非 root 用户，安装时需加 sudo。安装完可以检查这个目录是否有了这个文件: + +```shell +root@slave11 ~/udf $ ls -l /usr/local/lib/libtaos* +-rw-r--r-- 1 root root 671344 May 24 22:54 /usr/local/lib/libtaospyudf.so +``` ### 接口定义 -当使用Python语言开发UDF时，需要实现规定的接口函数。具体要求如下。 -- 标量函数需要实现标量接口函数process。 -- 聚合函数需要实现聚合接口函数start、reduce、finish。 -- 如果需要初始化，则应实现函数init。 -- 如果需要清理工作，则实现函数destroy。 +当使用 Python 语言开发 UDF 时，需要实现规定的接口函数。具体要求如下。 +- 标量函数需要实现标量接口函数 process。 +- 聚合函数需要实现聚合接口函数 start、reduce、finish。 +- 如果需要初始化，则应实现函数 init。 +- 如果需要清理工作，则实现函数 destroy。 #### 标量函数接口 @@ -147,7 +323,7 @@ def process(input: datablock) -> tuple[output_type]: ``` 主要参数说明如下： -- input:datablock 类似二维矩阵，通过成员方法 data(row,col)返回位于 row 行，col 列的 python 对象 +- input:datablock 类似二维矩阵，通过成员方法 data(row, col) 读取位于 row 行、col 列的 python 对象 - 返回值是一个 Python 对象元组，每个元素类型为输出类型。 #### 聚合函数接口 @@ -159,13 +335,13 @@ def reduce(inputs: datablock, buf: bytes) -> bytes def finish(buf: bytes) -> output_type: ``` -上述代码定义了3个函数，分别用于实现一个自定义的聚合函数。具体过程如下。 +上述代码定义了 3 个函数，分别用于实现一个自定义的聚合函数。具体过程如下。 -首先，调用start函数生成最初的结果缓冲区。这个结果缓冲区用于存储聚合函数的内部状态，随着输入数据的处理而不断更新。 +首先，调用 start 函数生成最初的结果缓冲区。这个结果缓冲区用于存储聚合函数的内部状态，随着输入数据的处理而不断更新。 -然后，输入数据会被分为多个行数据块。对于每个行数据块，调用reduce函数，并将当前行数据块（inputs）和当前的中间结果（buf）作为参数传递。reduce函数会根据输入数据和当前状态来更新聚合函数的内部状态，并返回新的中间结果 +然后，输入数据会被分为多个行数据块。对于每个行数据块，调用 reduce 函数，并将当前行数据块（inputs）和当前的中间结果（buf）作为参数传递。reduce 函数会根据输入数据和当前状态来更新聚合函数的内部状态，并返回新的中间结果。 -最后，当所有行数据块都处理完毕后，调用finish函数。这个函数接收最终的中间结果（buf）作为参数，并从中生成最终的输出。由于聚合函数的特性，最终输出只能包含0条或1条数据。这个输出结果将作为聚合函数的计算结果返回给调用者。 +最后，当所有行数据块都处理完毕后，调用 finish 函数。这个函数接收最终的中间结果（buf）作为参数，并从中生成最终的输出。由于聚合函数的特性，最终输出只能包含 0 条或 1 条数据。这个输出结果将作为聚合函数的计算结果返回给调用者。 #### 初始化和销毁接口 @@ -179,7 +355,7 @@ def destroy() - init 完成初始化工作 - destroy 完成清理工作 -**注意** 用Python开发UDF时必须定义init函数和destroy函数 +**注意** 用 Python 开发 UDF 时必须定义 init 函数和 destroy 函数 ### 标量函数模板 @@ -204,7 +380,7 @@ def start() -> bytes: def reduce(inputs: datablock, buf: bytes) -> bytes # deserialize buf to state # reduce the inputs and state into new_state. - # use inputs.data(i,j) to access python object of location(i,j) + # use inputs.data(i, j) to access python object of location(i, j) # serialize new_state into new_state_bytes return new_state_bytes def finish(buf: bytes) -> output_type: @@ -217,13 +393,13 @@ def finish(buf: bytes) -> output_type: | **TDengine SQL数据类型** | **Python数据类型** | | :-----------------------: | ------------ | -|TINYINT / SMALLINT / INT / BIGINT | int | -|TINYINT UNSIGNED / SMALLINT UNSIGNED / INT UNSIGNED / BIGINT UNSIGNED | int | -|FLOAT / DOUBLE | float | -|BOOL | bool | -|BINARY / VARCHAR / NCHAR | bytes| -|TIMESTAMP | int | -|JSON and other types | 不支持 | +| TINYINT / SMALLINT / INT / BIGINT | int | +| TINYINT UNSIGNED / SMALLINT UNSIGNED / INT UNSIGNED / BIGINT UNSIGNED | int | +| FLOAT / DOUBLE | float | +| BOOL | bool | +| BINARY / VARCHAR / NCHAR | bytes| +| TIMESTAMP | int | +| JSON and other types | 不支持 | ### 开发示例 @@ -262,7 +438,7 @@ create function myfun as '/root/udf/myfun.py' outputtype double language 'Python 其输出如下 ```shell - taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; +taos> create function myfun as '/root/udf/myfun.py' outputtype double language 'Python'; Create OK, 0 row(s) affected (0.005202s) ``` @@ -460,7 +636,7 @@ def process(block): for i in range(rows)] ``` -UDF 框架会将 TDengine 的 timestamp 类型映射为 Python 的 int 类型，所以这个函数只接受一个表示毫秒数的整数。process 方法先做参数检查，然后用 moment 包替换时间的星期为星期日，最后格式化输出。输出的字符串长度是固定的10个字符长，因此可以这样创建 UDF 函数： +UDF 框架会将 TDengine 的 timestamp 类型映射为 Python 的 int 类型，所以这个函数只接受一个表示毫秒数的整数。process 方法先做参数检查，然后用 moment 包替换时间的星期为星期日，最后格式化输出。输出的字符串长度是固定的 10 个字符长，因此可以这样创建 UDF 函数： ```sql create function nextsunday as '/root/udf/nextsunday.py' outputtype binary(10) language 'Python'; @@ -627,39 +803,77 @@ close log file: spread.log 通过这个示例，我们学会了如何定义聚合函数，并打印自定义的日志信息。 +### 更多 Python UDF 示例代码 +#### 标量函数示例 [pybitand](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pybitand.py) + +pybitand 实现多列的按位与功能。如果只有一列，返回这一列。pybitand 忽略空值。 + +
+pybitand.py + +```Python +{{#include tests/script/sh/pybitand.py}} +``` + +
+ +#### 聚合函数示例 [pyl2norm](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pyl2norm.py) + +pyl2norm 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 + +
+pyl2norm.py + +```c +{{#include tests/script/sh/pyl2norm.py}} +``` + +
+ +#### 聚合函数示例 [pycumsum](https://github.com/taosdata/TDengine/blob/3.0/tests/script/sh/pycumsum.py) + +pycumsum 使用 numpy 计算输入列所有数据的累积和。 +
+pycumsum.py + +```c +{{#include tests/script/sh/pycumsum.py}} +``` + +
+ ## 管理 UDF -在集群中管理UDF的过程涉及创建、使用和维护这些函数。用户可以通过SQL在集群中创建和管理UDF，一旦创建成功，集群的所有用户都可以在SQL中使用这些函数。由于UDF存储在集群的mnode上，因此即使重启集群，已经创建的UDF也仍然可用。 +在集群中管理 UDF 的过程涉及创建、使用和维护这些函数。用户可以通过 SQL 在集群中创建和管理 UDF，一旦创建成功，集群的所有用户都可以在 SQL 中使用这些函数。由于 UDF 存储在集群的 mnode 上，因此即使重启集群，已经创建的 UDF 也仍然可用。 -在创建UDF时，需要区分标量函数和聚合函数。标量函数接受零个或多个输入参数，并返回一个单一的值。聚合函数接受一组输入值，并通过对这些值进行某种计算（如求和、计数等）来返回一个单一的值。如果创建时声明了错误的函数类别，则通过SQL调用函数时会报错。 +在创建 UDF 时，需要区分标量函数和聚合函数。标量函数接受零个或多个输入参数，并返回一个单一的值。聚合函数接受一组输入值，并通过对这些值进行某种计算（如求和、计数等）来返回一个单一的值。如果创建时声明了错误的函数类别，则通过 SQL 调用函数时会报错。 -此外，用户需要确保输入数据类型与UDF程序匹配，UDF输出的数据类型与outputtype匹配。这意味着在创建UDF时，需要为输入参数和输出值指定正确的数据类型。这有助于确保在调用UDF时，输入数据能够正确地传递给UDF，并且UDF的输出值与预期的数据类型相匹配。 +此外，用户需要确保输入数据类型与 UDF 程序匹配，UDF 输出的数据类型与 outputtype 匹配。这意味着在创建 UDF 时，需要为输入参数和输出值指定正确的数据类型。这有助于确保在调用 UDF 时，输入数据能够正确地传递给 UDF，并且 UDF 的输出值与预期的数据类型相匹配。 ### 创建标量函数 -创建标量函数的SQL语法如下。 +创建标量函数的 SQL 语法如下。 ```sql -CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +CREATE OR REPLACE FUNCTION function_name AS library_path OUTPUTTYPE output_type LANGUAGE 'Python'; ``` 各参数说明如下。 - or replace：如果函数已经存在，则会修改已有的函数属性。 - function_name：标量函数在SQL中被调用时的函数名。 -- language：支持C语言和Python语言（3.7及以上版本），默认为C。 -- library_path：如果编程语言是C，则路径是包含UDF实现的动态链接库的库文件绝对路径，通常指向一个so文件。如果编程语言是Python，则路径是包含UDF -实现的Python文件路径。路径需要用英文单引号或英文双引号括起来。 +- language：支持 C 语言和 Python 语言（3.7 及以上版本），默认为 C。 +- library_path：如果编程语言是 C，则路径是包含 UDF 实现的动态链接库的库文件绝对路径，通常指向一个 so 文件。如果编程语言是 Python，则路径是包含 UDF +实现的 Python 文件路径。路径需要用英文单引号或英文双引号括起来。 - output_type：函数计算结果的数据类型名称。 - ### 创建聚合函数 -创建聚合函数的SQL语法如下。 +创建聚合函数的 SQL 语法如下。 ```sql -CREATE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; +CREATE OR REPLACE AGGREGATE FUNCTION function_name library_path OUTPUTTYPE output_type LANGUAGE 'Python'; ``` 其中，buffer_size 表示中间计算结果的缓冲区大小，单位是字节。其他参数的含义与标量函数相同。 -如下SQL创建一个名为 l2norm 的UDF。 +如下 SQL 创建一个名为 l2norm 的 UDF。 ```sql CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 8; ``` @@ -673,8 +887,15 @@ DROP FUNCTION function_name; ### 查看 UDF -显示集群中当前可用的所有UDF的SQL如下。 +显示集群中当前可用的所有 UDF 的 SQL 如下。 ```sql show functions; ``` +### 查看函数信息 + +同名的 UDF 每更新一次，版本号会增加 1。 +```sql +select * from ins_functions \G; +``` + diff --git a/docs/zh/14-reference/01-components/01-taosd.md b/docs/zh/14-reference/01-components/01-taosd.md index d85839e62f..994f557a17 100644 --- a/docs/zh/14-reference/01-components/01-taosd.md +++ b/docs/zh/14-reference/01-components/01-taosd.md @@ -27,42 +27,42 @@ taosd 命令行参数如下 ### 连接相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|firstEp | taosd 启动时，主动连接的集群中首个 dnode 的 end point，缺省值：localhost:6030 | -|secondEp | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无| -|fqdn | 启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname | -|serverPort | 启动 taosd 后所监听的端口，缺省值：6030 | -|maxShellConns | 一个 dnode 容许的连接数，取值范围为 10-5000000，缺省值：5000 | -|numOfRpcSessions | 允许一个客户端能创建的最大连接数，取值范围 100-100000，缺省值：30000 | -|timeToGetAvailableConn | 获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000 | +| 参数名称 | 参数说明 | +| :--------------------: | :-------------------------------------------------------------------------------------: | +| firstEp | taosd 启动时，主动连接的集群中首个 dnode 的 end point，缺省值：localhost:6030 | +| secondEp | taosd 启动时，如果 firstEp 连接不上，尝试连接集群中第二个 dnode 的 endpoint，缺省值：无 | +| fqdn | 启动 taosd 后所监听的服务地址，缺省值：所在服务器上配置的第一个 hostname | +| serverPort | 启动 taosd 后所监听的端口，缺省值：6030 | +| maxShellConns | 一个 dnode 容许的连接数，取值范围为 10-5000000，缺省值：5000 | +| numOfRpcSessions | 允许一个客户端能创建的最大连接数，取值范围 100-100000，缺省值：30000 | +| timeToGetAvailableConn | 获得可用连接的最长等待时间，取值范围 10-50000000，单位为毫秒，缺省值：500000 | ### 监控相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|monitor | 是否收集监控数据并上报，0: 关闭；1:打开；缺省值：0 | -|monitorFqdn | taosKeeper 服务所在服务器的 FQDN，缺省值：无 | -|monitorPort | taosKeeper 服务所监听的端口号，缺省值：6043 | -|monitorInternal | 监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，缺省值：30| -|telemetryReporting | 是否上传 telemetry，0: 不上传，1：上传，缺省值：1 | -|crashReporting | 是否上传 crash 信息；0: 不上传，1: 上传；缺省值： 1| +| 参数名称 | 参数说明 | +| :----------------: | :------------------------------------------------------------------------------------: | +| monitor | 是否收集监控数据并上报，0: 关闭；1:打开；缺省值：0 | +| monitorFqdn | taosKeeper 服务所在服务器的 FQDN，缺省值：无 | +| monitorPort | taosKeeper 服务所监听的端口号，缺省值：6043 | +| monitorInternal | 监控数据库记录系统参数（CPU/内存）的时间间隔，单位是秒，取值范围 1-200000 ，缺省值：30 | +| telemetryReporting | 是否上传 telemetry，0: 不上传，1：上传，缺省值：1 | +| crashReporting | 是否上传 crash 信息；0: 不上传，1: 上传；缺省值： 1 | ### 查询相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|queryPolicy | 查询策略，1: 只使用 vnode，不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子，其余算子均在 qnode 执行 ；缺省值：1 | -|maxNumOfDistinctRes | 允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿 | -|countAlwaysReturnValue | ount/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值，0: 返回空行，1: 返回；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了TSMA时, 且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果. 注意此参数客户端和服务端值应保持一致.| +| 参数名称 | 参数说明 | +| :--------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| queryPolicy | 查询策略，1: 只使用 vnode，不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行，带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子，其余算子均在 qnode 执行 ；缺省值：1 | +| maxNumOfDistinctRes | 允许返回的 distinct 结果最大行数，默认值 10 万，最大允许值 1 亿 | +| countAlwaysReturnValue | ount/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值，0: 返回空行，1: 返回；该参数设置为 1 时，如果查询中含有 INTERVAL 子句或者该查询使用了TSMA时, 且相应的组或窗口内数据为空或者NULL， 对应的组或窗口将不返回查询结果. 注意此参数客户端和服务端值应保持一致. | ### 区域相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|timezone | 时区，缺省值：当前服务器所配置的时区 | -|locale | 系统区位信息及编码格式 ，缺省值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过 API 设置 | -|charset | 字符集编码，缺省值：系统自动获取 | +| 参数名称 | 参数说明 | +| :------: | :------------------------------------------------------------------------------------------------------: | +| timezone | 时区，缺省值：当前服务器所配置的时区 | +| locale | 系统区位信息及编码格式 ，缺省值：系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过 API 设置 | +| charset | 字符集编码，缺省值：系统自动获取 | :::info 1. 为应对多时区的数据写入和查询问题，TDengine 采用 Unix 时间戳(Unix Timestamp)来记录和存储时间戳。Unix 时间戳的特点决定了任一时刻不论在任何时区，产生的时间戳均一致。需要注意的是，Unix 时间戳是在客户端完成转换和记录。为了确保客户端其他形式的时间转换为正确的 Unix 时间戳，需要设置正确的时区。 @@ -141,54 +141,54 @@ charset 的有效值是 UTF-8。 ### 存储相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|dataDir | 数据文件目录，所有的数据文件都将写入该目录，缺省值：/var/lib | -|tempDir | 指定所有系统运行过程中的临时文件生成的目录，缺省值：/tmp | -|minimalTmpDirGB | tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值: 1| -|minimalDataDirGB | dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，缺省值: 2 | +| 参数名称 | 参数说明 | +| :--------------: | :--------------------------------------------------------------------: | +| dataDir | 数据文件目录，所有的数据文件都将写入该目录，缺省值：/var/lib | +| tempDir | 指定所有系统运行过程中的临时文件生成的目录，缺省值：/tmp | +| minimalTmpDirGB | tempDir 所指定的临时文件目录所需要保留的最小空间，单位 GB，缺省值: 1 | +| minimalDataDirGB | dataDir 指定的时序数据存储目录所需要保留的最小空间，单位 GB，缺省值: 2 | ### 集群相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|supportVnodes | dnode 支持的最大 vnode 数目，取值范围：0-4096，缺省值： CPU 核数的 2 倍 + 5 | +| 参数名称 | 参数说明 | +| :-----------: | :-------------------------------------------------------------------------: | +| supportVnodes | dnode 支持的最大 vnode 数目，取值范围：0-4096，缺省值： CPU 核数的 2 倍 + 5 | ### 性能调优 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 | +| 参数名称 | 参数说明 | +| :----------------: | :---------------------------------------------: | +| numOfCommitThreads | 写入线程的最大数量，取值范围 0-1024，缺省值为 4 | ### 日志相关 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|logDir | 日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos | -|minimalLogDirGB | 当日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位GB，缺省值：1| -|numOfLogLines | 单个日志文件允许的最大行数，缺省值：10,000,000 | -|asyncLog | 日志写入模式，0: 同步，1: 异步，缺省值: 1 | -|logKeepDays | 日志文件的最长保存时间 ，单位：天，缺省值：0，意味着无限保存；当设置为大于0 的值时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳。 | -|slowLogThreshold | 慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值: 3 | -|slowLogScope | 定启动记录哪些类型的慢查询，可选值：ALL, QUERY, INSERT, OHTERS, NONE; 默认值：ALL | -|debugFlag | 运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）; 默认值：131 或 135 （取决于不同模块）| -|tmrDebugFlag | 定时器模块的日志开关，取值范围同上 | -|uDebugFlag | 共用功能模块的日志开关，取值范围同上 | -|rpcDebugFlag | rpc 模块的日志开关，取值范围同上 | -|jniDebugFlag | jni 模块的日志开关，取值范围同上 | -|qDebugFlag | query 模块的日志开关，取值范围同上 | -|dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 | -|vDebugFlag | vnode 模块的日志开关，取值范围同上 | -|mDebugFlag | mnode 模块的日志开关，取值范围同上 | -|wDebugFlag | wal 模块的日志开关，取值范围同上 | -|sDebugFlag | sync 模块的日志开关，取值范围同上 | -|tsdbDebugFlag | tsdb 模块的日志开关，取值范围同上 | -|tqDebugFlag | tq 模块的日志开关，取值范围同上 | -|fsDebugFlag | fs 模块的日志开关，取值范围同上 | -|udfDebugFlag | udf 模块的日志开关，取值范围同上 | -|smaDebugFlag | sma 模块的日志开关，取值范围同上 | -|idxDebugFlag | index 模块的日志开关，取值范围同上 | -|tdbDebugFlag | tdb 模块的日志开关，取值范围同上 | +| 参数名称 | 参数说明 | +| :--------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------: | +| logDir | 日志文件目录，运行日志将写入该目录，缺省值：/var/log/taos | +| minimalLogDirGB | 当日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位GB，缺省值：1 | +| numOfLogLines | 单个日志文件允许的最大行数，缺省值：10,000,000 | +| asyncLog | 日志写入模式，0: 同步，1: 异步，缺省值: 1 | +| logKeepDays | 日志文件的最长保存时间 ，单位：天，缺省值：0，意味着无限保存；当设置为大于0 的值时，日志文件会被重命名为 taosdlog.xxx，其中 xxx 为日志文件最后修改的时间戳。 | +| slowLogThreshold | 慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值: 3 | +| slowLogScope | 定启动记录哪些类型的慢查询，可选值：ALL, QUERY, INSERT, OHTERS, NONE; 默认值：ALL | +| debugFlag | 运行日志开关，131（输出错误和警告日志），135（输出错误、警告和调试日志），143（输出错误、警告、调试和跟踪日志）; 默认值：131 或 135 （取决于不同模块） | +| tmrDebugFlag | 定时器模块的日志开关，取值范围同上 | +| uDebugFlag | 共用功能模块的日志开关，取值范围同上 | +| rpcDebugFlag | rpc 模块的日志开关，取值范围同上 | +| jniDebugFlag | jni 模块的日志开关，取值范围同上 | +| qDebugFlag | query 模块的日志开关，取值范围同上 | +| dDebugFlag | dnode 模块的日志开关，取值范围同上，缺省值 135 | +| vDebugFlag | vnode 模块的日志开关，取值范围同上 | +| mDebugFlag | mnode 模块的日志开关，取值范围同上 | +| wDebugFlag | wal 模块的日志开关，取值范围同上 | +| sDebugFlag | sync 模块的日志开关，取值范围同上 | +| tsdbDebugFlag | tsdb 模块的日志开关，取值范围同上 | +| tqDebugFlag | tq 模块的日志开关，取值范围同上 | +| fsDebugFlag | fs 模块的日志开关，取值范围同上 | +| udfDebugFlag | udf 模块的日志开关，取值范围同上 | +| smaDebugFlag | sma 模块的日志开关，取值范围同上 | +| idxDebugFlag | index 模块的日志开关，取值范围同上 | +| tdbDebugFlag | tdb 模块的日志开关，取值范围同上 | ### 压缩参数 @@ -216,10 +216,214 @@ lossyColumns float|double ### 其他参数 -| 参数名称 | 参数说明 | -|:-------------:|:----------------------------------------------------------------:| -|enableCoreFile | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值 为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下
2、手动启动，就在 taosd 执行目录下。| -|udf | 是否启动 UDF 服务；0: 不启动，1：启动；默认值 为 0 | -|ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变 ；默认值 为 | -| tmqMaxTopicNum| 订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值 为20 | -|maxTsmaNum | 集群内可创建的TSMA个数；取值范围：0-3；缺省值: 3| \ No newline at end of file +| 参数名称 | 参数说明 | +| :--------------: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | +| enableCoreFile | crash 时是否生成 core 文件；0: 不生成，1：生成；默认值 为 1; 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下
2、手动启动，就在 taosd 执行目录下。 | +| udf | 是否启动 UDF 服务；0: 不启动，1：启动；默认值 为 0 | +| ttlChangeOnWrite | ttl 到期时间是否伴随表的修改操作改变; 0: 不改变，1：改变 ；默认值 为 | +| tmqMaxTopicNum | 订阅最多可建立的 topic 数量; 取值范围 1-10000；缺省值 为20 | +| maxTsmaNum | 集群内可创建的TSMA个数；取值范围：0-3；缺省值: 3 | + + +## taosd 监控指标 + +taosd 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。 + +### taosd\_cluster\_basic 表 + +`taosd_cluster_basic` 表记录集群基础信息。 + +| field | type | is\_tag | comment | +| :------------------- | :-------- | :------ | :------------------------------ | +| ts | TIMESTAMP | | timestamp | +| first\_ep | VARCHAR | | 集群 first ep | +| first\_ep\_dnode\_id | INT | | 集群 first ep 的 dnode id | +| cluster_version | VARCHAR | | tdengine version。例如：3.0.4.0 | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_cluster\_info 表 + +`taosd_cluster_info` 表记录集群信息。 + +| field | type | is\_tag | comment | +| :----------------------- | :-------- | :------ | :----------------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| cluster_uptime | DOUBLE | | 当前 master 节点的uptime。单位：秒 | +| dbs\_total | DOUBLE | | database 总数 | +| tbs\_total | DOUBLE | | 当前集群 table 总数 | +| stbs\_total | DOUBLE | | 当前集群 stable 总数 | +| dnodes\_total | DOUBLE | | 当前集群 dnode 总数 | +| dnodes\_alive | DOUBLE | | 当前集群 dnode 存活总数 | +| mnodes\_total | DOUBLE | | 当前集群 mnode 总数 | +| mnodes\_alive | DOUBLE | | 当前集群 mnode 存活总数 | +| vgroups\_total | DOUBLE | | 当前集群 vgroup 总数 | +| vgroups\_alive | DOUBLE | | 当前集群 vgroup 存活总数 | +| vnodes\_total | DOUBLE | | 当前集群 vnode 总数 | +| vnodes\_alive | DOUBLE | | 当前集群 vnode 存活总数 | +| connections\_total | DOUBLE | | 当前集群连接总数 | +| topics\_total | DOUBLE | | 当前集群 topic 总数 | +| streams\_total | DOUBLE | | 当前集群 stream 总数 | +| grants_expire\_time | DOUBLE | | 认证过期时间，企业版有效，社区版为 DOUBLE 最大值 | +| grants_timeseries\_used | DOUBLE | | 已用测点数 | +| grants_timeseries\_total | DOUBLE | | 总测点数，开源版本为 DOUBLE 最大值 | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_vgroups\_info 表 + +`taosd_vgroups_info` 表记录虚拟节点组信息。 + +| field | type | is\_tag | comment | +| :------------- | :-------- | :------ | :--------------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| tables\_num | DOUBLE | | vgroup 中 table 数量 | +| status | DOUBLE | | vgroup 状态, 取值范围：unsynced = 0, ready = 1 | +| vgroup\_id | VARCHAR | TAG | vgroup id | +| database\_name | VARCHAR | TAG | vgroup 所属的 database 名字 | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_dnodes\_info 表 + +`taosd_dnodes_info` 记录 dnode 信息。 + +| field | type | is\_tag | comment | +| :---------------- | :-------- | :------ | :------------------------------------------------------------------------------------------------ | +| \_ts | TIMESTAMP | | timestamp | +| uptime | DOUBLE | | dnode uptime，单位：秒 | +| cpu\_engine | DOUBLE | | taosd cpu 使用率，从 `/proc//stat` 读取 | +| cpu\_system | DOUBLE | | 服务器 cpu 使用率，从 `/proc/stat` 读取 | +| cpu\_cores | DOUBLE | | 服务器 cpu 核数 | +| mem\_engine | DOUBLE | | taosd 内存使用率，从 `/proc//status` 读取 | +| mem\_free | DOUBLE | | 服务器可用内存，单位 KB | +| mem\_total | DOUBLE | | 服务器内存总量，单位 KB | +| disk\_used | DOUBLE | | data dir 挂载的磁盘使用量，单位 bytes | +| disk\_total | DOUBLE | | data dir 挂载的磁盘总容量，单位 bytes | +| system\_net\_in | DOUBLE | | 网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 byte/s | +| system\_net\_out | DOUBLE | | 网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 byte/s | +| io\_read | DOUBLE | | io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 byte/s | +| io\_write | DOUBLE | | io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 byte/s | +| io\_read\_disk | DOUBLE | | 磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 byte/s | +| io\_write\_disk | DOUBLE | | 磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 byte/s | +| vnodes\_num | DOUBLE | | dnode 上 vnodes 数量 | +| masters | DOUBLE | | dnode 上 master node 数量 | +| has\_mnode | DOUBLE | | dnode 是否包含 mnode，取值范围：包含=1,不包含=0 | +| has\_qnode | DOUBLE | | dnode 是否包含 qnode，取值范围：包含=1,不包含=0 | +| has\_snode | DOUBLE | | dnode 是否包含 snode，取值范围：包含=1,不包含=0 | +| has\_bnode | DOUBLE | | dnode 是否包含 bnode，取值范围：包含=1,不包含=0 | +| error\_log\_count | DOUBLE | | error 总数 | +| info\_log\_count | DOUBLE | | info 总数 | +| debug\_log\_count | DOUBLE | | debug 总数 | +| trace\_log\_count | DOUBLE | | trace 总数 | +| dnode\_id | VARCHAR | TAG | dnode id | +| dnode\_ep | VARCHAR | TAG | dnode endpoint | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_dnodes\_status 表 + +`taosd_dnodes_status` 表记录 dnode 状态信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :--------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| status | DOUBLE | | dnode 状态,取值范围：ready=1，offline =0 | +| dnode\_id | VARCHAR | TAG | dnode id | +| dnode\_ep | VARCHAR | TAG | dnode endpoint | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_dnodes\_log\_dir 表 + +`taosd_dnodes_log_dir` 表记录 log 目录信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :---------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| avail | DOUBLE | | log 目录可用空间。单位 byte | +| used | DOUBLE | | log 目录已使用空间。单位 byte | +| total | DOUBLE | | log 目录空间。单位 byte | +| name | VARCHAR | TAG | log 目录名，一般为 `/var/log/taos/` | +| dnode\_id | VARCHAR | TAG | dnode id | +| dnode\_ep | VARCHAR | TAG | dnode endpoint | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_dnodes\_data\_dir 表 + +`taosd_dnodes_data_dir` 表记录 data 目录信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :-------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| avail | DOUBLE | | data 目录可用空间。单位 byte | +| used | DOUBLE | | data 目录已使用空间。单位 byte | +| total | DOUBLE | | data 目录空间。单位 byte | +| level | VARCHAR | TAG | 0、1、2 多级存储级别 | +| name | VARCHAR | TAG | data 目录，一般为 `/var/lib/taos` | +| dnode\_id | VARCHAR | TAG | dnode id | +| dnode\_ep | VARCHAR | TAG | dnode endpoint | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_mnodes\_info 表 + +`taosd_mnodes_info` 表记录 mnode 角色信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :------------------------------------------------------------------------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| role | DOUBLE | | mnode 角色， 取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 | +| mnode\_id | VARCHAR | TAG | master node id | +| mnode\_ep | VARCHAR | TAG | master node endpoint | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_vnodes\_role 表 + +`taosd_vnodes_role` 表记录虚拟节点角色信息。 + +| field | type | is\_tag | comment | +| :------------- | :-------- | :------ | :------------------------------------------------------------------------------------------------------ | +| \_ts | TIMESTAMP | | timestamp | +| vnode\_role | DOUBLE | | vnode 角色，取值范围：offline = 0,follower = 100,candidate = 101,leader = 102,error = 103,learner = 104 | +| vgroup\_id | VARCHAR | TAG | dnode id | +| dnode\_id | VARCHAR | TAG | dnode id | +| database\_name | VARCHAR | TAG | vgroup 所属的 database 名字 | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taosd\_sql\_req 表 + +`taosd_sql_req` 记录服务端 sql 请求信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :--------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| count | DOUBLE | | sql 数量 | +| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed | +| username | VARCHAR | TAG | 执行sql的user name | +| sql\_type | VARCHAR | TAG | sql类型，取值范围：inserted_rows | +| dnode\_id | VARCHAR | TAG | dnode id | +| dnode\_ep | VARCHAR | TAG | dnode endpoint | +| vgroup\_id | VARCHAR | TAG | dnode id | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taos\_sql\_req 表 + +`taos_sql_req` 记录客户端 sql 请求信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :---------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| count | DOUBLE | | sql 数量 | +| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed | +| username | VARCHAR | TAG | 执行sql的user name | +| sql\_type | VARCHAR | TAG | sql类型，取值范围：select, insert，delete | +| cluster\_id | VARCHAR | TAG | cluster id | + +### taos\_slow\_sql 表 + +`taos_slow_sql` 记录客户端慢查询信息。 + +| field | type | is\_tag | comment | +| :---------- | :-------- | :------ | :---------------------------------------------------- | +| \_ts | TIMESTAMP | | timestamp | +| count | DOUBLE | | sql 数量 | +| result | VARCHAR | TAG | sql的执行结果，取值范围：Success, Failed | +| username | VARCHAR | TAG | 执行sql的user name | +| duration | VARCHAR | TAG | sql执行耗时，取值范围：3-10s,10-100s,100-1000s,1000s- | +| cluster\_id | VARCHAR | TAG | cluster id | + diff --git a/docs/zh/14-reference/01-components/03-taosadapter.md b/docs/zh/14-reference/01-components/03-taosadapter.md index 06ba4a9a07..84facad093 100644 --- a/docs/zh/14-reference/01-components/03-taosadapter.md +++ b/docs/zh/14-reference/01-components/03-taosadapter.md @@ -289,31 +289,32 @@ http 返回内容： ## taosAdapter 监控指标 -taosAdapter 采集 http 相关指标、CPU 百分比和内存百分比。 +taosAdapter 采集 REST/Websocket 相关请求的监控指标。将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。 -### http 接口 +#### adapter\_requests 表 -提供符合 [OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md) 接口： +`adapter_requests` 记录 taosadapter 监控数据。 -```text -http://:6041/metrics -``` - -### 写入 TDengine - -taosAdapter 支持将 http 监控、CPU 百分比和内存百分比写入 TDengine。 - -有关配置参数 - -| **配置项** | **描述** | **默认值** | -|-------------------------|--------------------------------------------|----------| -| monitor.collectDuration | CPU 和内存采集间隔 | 3s | -| monitor.identity | 当前taosadapter 的标识符如果不设置将使用 'hostname:port' | | -| monitor.incgroup | 是否是 cgroup 中运行(容器中运行设置为 true) | false | -| monitor.writeToTD | 是否写入到 TDengine | false | -| monitor.user | TDengine 连接用户名 | root | -| monitor.password | TDengine 连接密码 | taosdata | -| monitor.writeInterval | 写入TDengine 间隔 | 30s | +| field | type | is\_tag | comment | +| :----------------- | :----------- | :------ | :---------------------------------- | +| ts | TIMESTAMP | | timestamp | +| total | INT UNSIGNED | | 总请求数 | +| query | INT UNSIGNED | | 查询请求数 | +| write | INT UNSIGNED | | 写入请求数 | +| other | INT UNSIGNED | | 其他请求数 | +| in\_process | INT UNSIGNED | | 正在处理请求数 | +| success | INT UNSIGNED | | 成功请求数 | +| fail | INT UNSIGNED | | 失败请求数 | +| query\_success | INT UNSIGNED | | 查询成功请求数 | +| query\_fail | INT UNSIGNED | | 查询失败请求数 | +| write\_success | INT UNSIGNED | | 写入成功请求数 | +| write\_fail | INT UNSIGNED | | 写入失败请求数 | +| other\_success | INT UNSIGNED | | 其他成功请求数 | +| other\_fail | INT UNSIGNED | | 其他失败请求数 | +| query\_in\_process | INT UNSIGNED | | 正在处理查询请求数 | +| write\_in\_process | INT UNSIGNED | | 正在处理写入请求数 | +| endpoint | VARCHAR | | 请求端点 | +| req\_type | NCHAR | TAG | 请求类型：0 为 REST，1 为 Websocket | ## 结果返回条数限制 @@ -342,11 +343,11 @@ taosAdapter 从 3.0.4.0 版本开始，提供参数 `smlAutoCreateDB` 来控制 在 TDengine server 2.2.x.x 或更早期版本中，taosd 进程包含一个内嵌的 http 服务。如前面所述，taosAdapter 是一个使用 systemd 管理的独立软件，拥有自己的进程。并且两者有一些配置参数和行为是不同的，请见下表： -| **#** | **embedded httpd** | **taosAdapter** | **comment** | -|-------|---------------------|-------------------------------|------------------------------------------------------------------------------------------------| -| 1 | httpEnableRecordSql | --logLevel=debug | | -| 2 | httpMaxThreads | n/a | taosAdapter 自动管理线程池，无需此参数 | +| **#** | **embedded httpd** | **taosAdapter** | **comment** | +| ----- | ------------------- | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 | httpEnableRecordSql | --logLevel=debug | | +| 2 | httpMaxThreads | n/a | taosAdapter 自动管理线程池，无需此参数 | | 3 | telegrafUseFieldNum | 请参考 taosAdapter telegraf 配置方法 | -| 4 | restfulRowLimit | restfulRowLimit | 内嵌 httpd 默认输出 10240 行数据，最大允许值为 102400。taosAdapter 也提供 restfulRowLimit 但是默认不做限制。您可以根据实际场景需求进行配置 | -| 5 | httpDebugFlag | 不适用 | httpdDebugFlag 对 taosAdapter 不起作用 | -| 6 | httpDBNameMandatory | 不适用 | taosAdapter 要求 URL 中必须指定数据库名 | +| 4 | restfulRowLimit | restfulRowLimit | 内嵌 httpd 默认输出 10240 行数据，最大允许值为 102400。taosAdapter 也提供 restfulRowLimit 但是默认不做限制。您可以根据实际场景需求进行配置 | +| 5 | httpDebugFlag | 不适用 | httpdDebugFlag 对 taosAdapter 不起作用 | +| 6 | httpDBNameMandatory | 不适用 | taosAdapter 要求 URL 中必须指定数据库名 | diff --git a/docs/zh/14-reference/01-components/04-taosx.md b/docs/zh/14-reference/01-components/04-taosx.md index 166b6ea760..e378c18800 100644 --- a/docs/zh/14-reference/01-components/04-taosx.md +++ b/docs/zh/14-reference/01-components/04-taosx.md @@ -90,11 +90,11 @@ taosx privileges -i ./user-pass-privileges-backup.json -t "taos:///" 可用参数列表： -| 参数 | 说明 | -| ---- | ---- | -| -u | 包含用户基本信息（密码、是否启用等） | -| -p | 包含权限信息 | -| -w | 包含白名单信息 | +| 参数 | 说明 | +| ---- | ------------------------------------ | +| -u | 包含用户基本信息（密码、是否启用等） | +| -p | 包含权限信息 | +| -w | 包含白名单信息 | 当 `-u`/`-p` 参数应用时，将仅包含指定的信息，不带参数时，表示所有信息（用户名、密码、权限和白名单）。 @@ -324,4 +324,131 @@ Linux 下 `journalctl` 查看日志的命令如下： ```bash journalctl -u taosx [-f] -``` \ No newline at end of file +``` + +## taosX 监控指标 + +taosX 会将监控指标上报给 taosKeeper，这些监控指标会被 taosKeeper 写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。以下是这些监控指标的详细介绍。 + +### taosX 服务 + +| 字段 | 描述 | +| -------------------------- | ----------------------------------------------------------------------------- | +| sys_cpu_cores | 系统 CPU 核数 | +| sys_total_memory | 系统总内存，单位：字节 | +| sys_used_memory | 系统已用内存, 单位：字节 | +| sys_available_memory | 系统可用内存, 单位：字节 | +| process_uptime | taosX 运行时长，单位：秒 | +| process_id | taosX 进程 ID | +| running_tasks | taosX 当前执行任务数 | +| completed_tasks | taosX 进程在一个监控周期（比如10s）内完成的任务数 | +| failed_tasks | taosX 进程在一个监控周期（比如10s）内失败的任务数 | +| process_cpu_percent | taosX 进程占用 CPU 百分比， 单位 % | +| process_memory_percent | taosX 进程占用内存百分比， 单位 % | +| process_disk_read_bytes | taosX 进程在一个监控周期（比如10s）内从硬盘读取的字节数的平均值，单位 bytes/s | +| process_disk_written_bytes | taosX 进程在一个监控周期（比如10s）内写到硬盘的字节数的平均值，单位 bytres/s | + + +### Agent + +| 字段 | 描述 | +| -------------------------- | ----------------------------------------------------------------------------- | +| sys_cpu_cores | 系统 CPU 核数 | +| sys_total_memory | 系统总内存，单位：字节 | +| sys_used_memory | 系统已用内存, 单位：字节 | +| sys_available_memory | 系统可用内存, 单位：字节 | +| process_uptime | agent 运行时长，单位：秒 | +| process_id | agent 进程 id | +| process_cpu_percent | agent 进程占用 CPU 百分比 | +| process_memory_percent | agent 进程占用内存百分比 | +| process_uptime | 进程启动时间，单位秒 | +| process_disk_read_bytes | agent 进程在一个监控周期（比如10s）内从硬盘读取的字节数的平均值，单位 bytes/s | +| process_disk_written_bytes | agent 进程在一个监控周期（比如10s）内写到硬盘的字节数的平均值，单位 bytes/s | + +### Connector + +| 字段 | 描述 | +| -------------------------- | --------------------------------------------------------------------------------- | +| process_id | connector 进程 id | +| process_uptime | 进程启动时间，单位秒 | +| process_cpu_percent | 进程占用 CPU 百分比， 单位 % | +| process_memory_percent | 进程占用内存百分比， 单位 % | +| process_disk_read_bytes | connector 进程在一个监控周期（比如10s）内从硬盘读取的字节数的平均值，单位 bytes/s | +| process_disk_written_bytes | connector 进程在一个监控周期（比如10s）内写到硬盘的字节数的平均值，单位 bytes/s | + +### taosX 通用数据源任务 + +| 字段 | 描述 | +| -------------------- | --------------------------------------------------------------- | +| total_execute_time | 任务累计运行时间，单位毫秒 | +| total_written_rowsls | 成功写入 TDengine 的总行数（包括重复记录） | +| total_written_points | 累计写入成功点数 (等于数据块包含的行数乘以数据块包含的列数) | +| start_time | 任务启动时间 (每次重启任务会被重置) | +| written_rows | 本次运行此任务成功写入 TDengine 的总行数（包括重复记录） | +| written_points | 本次运行写入成功点数 (等于数据块包含的行数乘以数据块包含的列数) | +| execute_time | 任务本次运行时间，单位秒 | + +### taosX TDengine V2 任务 + +| 字段 | 描述 | +| --------------------- | -------------------------------------------------------------------- | +| read_concurrency | 并发读取数据源的数据 worker 数, 也等于并发写入 TDengine 的 worker 数 | +| total_stables | 需要迁移的超级表数据数量 | +| total_updated_tags | 累计更新 tag 数 | +| total_created_tables | 累计创建子表数 | +| total_tables | 需要迁移的子表数量 | +| total_finished_tables | 完成数据迁移的子表数 (任务中断重启可能大于实际值) | +| total_success_blocks | 累计写入成功的数据块数 | +| finished_tables | 本次运行完成迁移子表数 | +| success_blocks | 本次写入成功的数据块数 | +| created_tables | 本次运行创建子表数 | +| updated_tags | 本次运行更新 tag 数 | + +### taosX TDengine V3 任务 + +| 字段 | 描述 | +| ---------------------- | ------------------------------------------------------- | +| total_messages | 通过 TMQ 累计收到的消息总数 | +| total_messages_of_meta | 通过 TMQ 累计收到的 Meta 类型的消息总数 | +| total_messages_of_data | 通过 TMQ 累计收到的 Data 和 MetaData 类型的消息总数 | +| total_write_raw_fails | 累计写入 raw meta 失败的次数 | +| total_success_blocks | 累计写入成功的数据块数 | +| topics | 通过 TMQ 订阅的主题数 | +| consumers | TMQ 消费者数 | +| messages | 本次运行通过 TMQ 收到的消息总数 | +| messages_of_meta | 本次运行通过 TMQ 收到的 Meta 类型的消息总数 | +| messages_of_data | 本次运行通过 TMQ 收到的 Data 和 MetaData 类型的消息总数 | +| write_raw_fails | 本次运行写入 raw meta 失败的次数 | +| success_blocks | 本次写入成功的数据块数 | + + +### taosX 其他数据源 任务 + +这些数据源包括： InfluxDB，OpenTSDB，OPC UA，OPC DA，PI，CSV，MQTT，AVEVA Historian 和 Kafka。 + +| 字段 | 描述 | +| ----------------------- | ----------------------------------------------------------- | +| total_received_batches | 通过 IPC Stream 收到的数据总批数 | +| total_processed_batches | 已经处理的批数 | +| total_processed_rows | 已经处理的总行数（等于每批包含数据行数之和） | +| total_inserted_sqls | 执行的 INSERT SQL 总条数 | +| total_failed_sqls | 执行失败的 INSERT SQL 总条数 | +| total_created_stables | 创建的超级表总数（可能大于实际值） | +| total_created_tables | 尝试创建子表总数(可能大于实际值) | +| total_failed_rows | 写入失败的总行数 | +| total_failed_point | 写入失败的总点数 | +| total_written_blocks | 写入成功的 raw block 总数 | +| total_failed_blocks | 写入失败的 raw block 总数 | +| received_batches | 本次运行此任务通过 IPC Stream 收到的数据总批数 | +| processed_batches | 本次运行已处理批数 | +| processed_rows | 本次处理的总行数（等于包含数据的 batch 包含的数据行数之和） | +| received_records | 本次运行此任务通过 IPC Stream 收到的数据总行数 | +| inserted_sqls | 本次运行此任务执行的 INSERT SQL 总条数 | +| failed_sqls | 本次运行此任务执行失败的 INSERT SQL 总条数 | +| created_stables | 本次运行此任务尝试创建超级表数（可能大于实际值） | +| created_tables | 本次运行此任务尝试创建子表数(可能大于实际值) | +| failed_rows | 本次运行此任务写入失败的行数 | +| failed_points | 本次运行此任务写入失败的点数 | +| written_blocks | 本次运行此任务写人成功的 raw block 数 | +| failed_blocks | 本次运行此任务写入失败的 raw block 数 | + diff --git a/docs/zh/14-reference/01-components/06-taoskeeper.md b/docs/zh/14-reference/01-components/06-taoskeeper.md index 69fd959f16..2877728077 100644 --- a/docs/zh/14-reference/01-components/06-taoskeeper.md +++ b/docs/zh/14-reference/01-components/06-taoskeeper.md @@ -14,17 +14,102 @@ taosKeeper 是 TDengine 3.0 版本监控指标的导出工具，通过简单的 ## 安装 taosKeeper 有两种安装方式： -taosKeeper 安装方式： - 安装 TDengine 官方安装包的同时会自动安装 taosKeeper, 详情请参考[ TDengine 安装](../../../get-started/)。 - 单独编译 taosKeeper 并安装，详情请参考 [taosKeeper](https://github.com/taosdata/taoskeeper) 仓库。 -## 配置和运行方式 +## 配置 -### 配置 +taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：命令行参数、环境变量 和 配置文件。优先级为：命令行参数、环境变量、配置文件参数。 一般我们推荐使用配置文件。 -taosKeeper 需要在操作系统终端执行，该工具支持三种配置方式：[命令行参数](#命令行参数启动)、[环境变量](#环境变量启动) 和 [配置文件](#配置文件启动)。优先级为：命令行参数、环境变量、配置文件参数。 +### 命令行参数和环境变量 +命令行参数 和 环境变量说明可以参考命令 `taoskeeper --help` 的输出。下面是一个例子： +```shell +Usage of taosKeeper v3.3.2.0: + --debug enable debug mode. Env "TAOS_KEEPER_DEBUG" + -P, --port int http port. Env "TAOS_KEEPER_PORT" (default 6043) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_KEEPER_LOG_LEVEL" (default "info") + --gopoolsize int coroutine size. Env "TAOS_KEEPER_POOL_SIZE" (default 50000) + -R, --RotationInterval string interval for refresh metrics, such as "300ms", Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Env "TAOS_KEEPER_ROTATION_INTERVAL" (default "15s") + --tdengine.host string TDengine server's ip. Env "TAOS_KEEPER_TDENGINE_HOST" (default "127.0.0.1") + --tdengine.port int TDengine REST server(taosAdapter)'s port. Env "TAOS_KEEPER_TDENGINE_PORT" (default 6041) + --tdengine.username string TDengine server's username. Env "TAOS_KEEPER_TDENGINE_USERNAME" (default "root") + --tdengine.password string TDengine server's password. Env "TAOS_KEEPER_TDENGINE_PASSWORD" (default "taosdata") + --tdengine.usessl TDengine server use ssl or not. Env "TAOS_KEEPER_TDENGINE_USESSL" + --metrics.prefix string prefix in metrics names. Env "TAOS_KEEPER_METRICS_PREFIX" + --metrics.database.name string database for storing metrics data. Env "TAOS_KEEPER_METRICS_DATABASE" (default "log") + --metrics.tables stringArray export some tables that are not super table, multiple values split with white space. Env "TAOS_KEEPER_METRICS_TABLES" + --environment.incgroup whether running in cgroup. Env "TAOS_KEEPER_ENVIRONMENT_INCGROUP" + --log.path string log path. Env "TAOS_KEEPER_LOG_PATH" (default "/var/log/taos") + --log.rotationCount uint log rotation count. Env "TAOS_KEEPER_LOG_ROTATION_COUNT" (default 5) + --log.rotationTime duration log rotation time. Env "TAOS_KEEPER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_KEEPER_LOG_ROTATION_SIZE" (default "100000000") + -c, --config string config path default /etc/taos/taoskeeper.toml + -V, --version Print the version and exit + -h, --help Print this help message and exit +``` + + + +### 配置文件 + +taosKeeper 支持用 `taoskeeper -c ` 命令来指定配置文件。 +若不指定配置文件，taosKeeper 会使用默认配置文件，其路径为： `/etc/taos/taoskeeper.toml` 。 +若既不指定 taosKeeper 配置文件，且 `/etc/taos/taoskeeper.toml` 也不存在，将使用默认配置。 + +**下面是配置文件的示例：** +```toml +# Start with debug middleware for gin +debug = false + +# Listen port, default is 6043 +port = 6043 + +# log level +loglevel = "info" + +# go pool size +gopoolsize = 50000 + +# interval for metrics +RotationInterval = "15s" + +[tdengine] +host = "127.0.0.1" +port = 6041 +username = "root" +password = "taosdata" +usessl = false + +[metrics] +# metrics prefix in metrics names. +prefix = "taos" + +# export some tables that are not super table +tables = [] + +# database for storing metrics data +[metrics.database] +name = "log" +# database options for db storing metrics data +[metrics.database.options] +vgroups = 1 +buffer = 64 +KEEP = 90 +cachemodel = "both" + +[environment] +# Whether running in cgroup. +incgroup = false + +[log] +rotationCount = 5 +rotationTime = "24h" +rotationSize = 100000000 +``` + +## 启动 **在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** 并且 TDengine 已经开启监控服务，TDengine 配置文件 `taos.cfg` 中至少需要配置 `monitor` 和 `monitorFqdn`。 @@ -36,8 +121,6 @@ monitorFqdn localhost # taoskeeper 服务的 FQDN TDengine 监控配置相关，具体请参考：[TDengine 监控配置](../../../operation/monitor)。 -### 启动 - @@ -79,8 +162,8 @@ Active: inactive (dead) - `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。 - 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taoskeeper` 方式启动 taoskeeper 服务。 -- 故障排查： -- 如果服务异常请查看系统日志获取更多信息。 +- 故障排查：如果服务异常请查看日志获取更多信息。日志文件默认放在 `/var/log/taos` 下。 + ::: @@ -100,8 +183,7 @@ Active: inactive (dead) - `launchctl` 命令管理`com.tdengine.taoskeeper`需要管理员权限，务必在前面加 `sudo` 来增强安全性。 - `sudo launchctl list | grep taoskeeper` 指令返回的第一列是 `taoskeeper` 程序的 PID，若为 `-` 则说明 taoskeeper 服务未运行。 -- 故障排查： -- 如果服务异常请查看系统日志获取更多信息。 +- 故障排查：如果服务异常请查看日志获取更多信息。日志文件默认放在 `/var/log/taos` 下。 ::: @@ -109,88 +191,82 @@ Active: inactive (dead) -#### 配置文件启动 +## 健康检查 -执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/taoskeeper.toml` 配置，否则将使用默认配置。 +可以访问 taosKeeper 的 `check_health` 接口来判断服务是否存活，如果服务正常则会返回 HTTP 200 状态码： +``` +$ curl -i http://127.0.0.1:6043/check_health +``` + +返回结果： + +``` +HTTP/1.1 200 OK +Content-Type: application/json; charset=utf-8 +Date: Wed, 07 Aug 2024 06:19:50 GMT +Content-Length: 21 + +{"version":"3.3.2.3"} +``` + + +## 数据收集与监控 + +taosKeeper 作为 TDengine 监控指标的导出工具，可以将 TDengine 产生的监控数据记录在指定数据库中（默认的监控数据是 `log`），这些监控数据可以用来配置 TDengine 监控。 + +### 查看监控数据 + +可以查看 `log` 库下的超级表，每个超级表都对应一组监控指标，具体指标不再赘述。 ```shell -$ taoskeeper -c -``` +taos> use log; +Database changed. -**下面是配置文件的示例：** -```toml -# gin 框架是否启用 debug -debug = false - -# 服务监听端口, 默认为 6043 -port = 6043 - -# 日志级别，包含 panic、error、info、debug、trace等 -loglevel = "info" - -# 程序中使用协程池的大小 -gopoolsize = 50000 - -# 查询 TDengine 监控数据轮询间隔 -RotationInterval = "15s" - -[tdengine] -host = "127.0.0.1" -port = 6041 -username = "root" -password = "taosdata" - -[metrics] -# 监控指标前缀 -prefix = "taos" - -# 存放监控数据的数据库 -database = "log" - -# 指定需要监控的普通表 -tables = [] - -# 监控数据的配置选项 -[metrics.databaseoptions] -cachemodel = "none" - -[environment] -# 容器模式收集信息 -incgroup = false - -[log] -# 日志文件滚动个数 -rotationCount = 5 -# 日志文件切割时间 -rotationTime = "24h" -# 日志文件切割大小 (字节) -rotationSize = 100000000 +taos> show stables; + stable_name | +================================= + taosd_dnodes_status | + taosd_vnodes_info | + keeper_monitor | + taosd_vgroups_info | + taos_sql_req | + taos_slow_sql | + taosd_mnodes_info | + taosd_cluster_info | + taosd_sql_req | + taosd_dnodes_info | + adapter_requests | + taosd_cluster_basic | + taosd_dnodes_data_dirs | + taosd_dnodes_log_dirs | +Query OK, 14 row(s) in set (0.006542s) ``` -### 获取监控指标 +可以查看一个超级表的最近一条上报记录，如： -taosKeeper 作为 TDengine 监控指标的导出工具，可以将 TDengine 产生的监控数据记录在指定数据库中，并提供导出接口。 - -#### 查看监控结果集 - -```shell -$ taos -# 如上示例，使用 log 库作为监控日志存储位置 -> use log; -> select * from taosd_cluster_info limit 1; +``` shell +taos> select last_row(*) from taosd_dnodes_info; + last_row(_ts) | last_row(disk_engine) | last_row(system_net_in) | last_row(vnodes_num) | last_row(system_net_out) | last_row(uptime) | last_row(has_mnode) | last_row(io_read_disk) | last_row(error_log_count) | last_row(io_read) | last_row(cpu_cores) | last_row(has_qnode) | last_row(has_snode) | last_row(disk_total) | last_row(mem_engine) | last_row(info_log_count) | last_row(cpu_engine) | last_row(io_write_disk) | last_row(debug_log_count) | last_row(disk_used) | last_row(mem_total) | last_row(io_write) | last_row(masters) | last_row(cpu_system) | last_row(trace_log_count) | last_row(mem_free) | +====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== + 2024-08-07 14:54:09.174 | 0.000000000000000 | 3379.093240947399863 | 37.000000000000000 | 5265.998201139278535 | 64402.000000000000000 | 1.000000000000000 | 8323.261934108399146 | 6.000000000000000 | 40547.386655118425551 | 16.000000000000000 | 0.000000000000000 | 0.000000000000000 | 5.272955781120000e+11 | 2443032.000000000000000 | 423.000000000000000 | 0.556269622200215 | 677731.836503547732718 | 356380.000000000000000 | 4.997186764800000e+10 | 65557284.000000000000000 | 714177.054532129666768 | 37.000000000000000 | 2.642280705451021 | 0.000000000000000 | 11604276.000000000000000 | +Query OK, 1 row(s) in set (0.003168s) ``` -结果示例： -```shell - _ts | cluster_uptime | dbs_total | tbs_total | stbs_total | vgroups_total | vgroups_alive | vnodes_total | vnodes_alive | mnodes_total | mnodes_alive | connections_total | topics_total | streams_total | dnodes_total | dnodes_alive | grants_expire_time | grants_timeseries_used | grants_timeseries_total | cluster_id | -=================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== - 2024-06-04 03:03:34.341 | 0.000000000000000 | 2.000000000000000 | 1.000000000000000 | 4.000000000000000 | 4.000000000000000 | 4.000000000000000 | 4.000000000000000 | 4.000000000000000 | 1.000000000000000 | 1.000000000000000 | 2.000000000000000 | 0.000000000000000 | 0.000000000000000 | 1.000000000000000 | 1.000000000000000 | 0.000000000000000 | 3.000000000000000 | 0.000000000000000 | 554014120921134497 | -Query OK, 1 row(s) in set (0.001652s) -``` +### 使用 TDInsight 配置监控 -#### 导出监控指标 +收集到监控数据以后，就可以使用 TDInsight 来配置 TDengine 的监控，具体请参考 [TDinsight 参考手册](../tdinsight/) + + +## 集成 Prometheus + +taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 + + +### 导出监控指标 + +下面通过 `curl` 命令展示 `/metrics` 接口返回的数据格式： ```shell $ curl http://127.0.0.1:6043/metrics @@ -219,28 +295,8 @@ taos_cluster_info_first_ep{cluster_id="554014120921134497",value="tdengine:6030" taos_cluster_info_first_ep_dnode_id{cluster_id="554014120921134497"} 1 ``` -### check\_health -``` -$ curl -i http://127.0.0.1:6043/check_health -``` - -返回结果： - -``` -HTTP/1.1 200 OK -Content-Type: application/json; charset=utf-8 -Date: Mon, 03 Apr 2023 07:20:38 GMT -Content-Length: 19 - -{"version":"1.0.0"} -``` - -### 集成 Prometheus - -taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 - -#### 抽取配置 +### 抽取配置 Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。 @@ -255,9 +311,25 @@ scrape_configs: - targets: ["localhost:6043"] ``` -#### Dashboard +### Dashboard 我们提供了 `TaosKeeper Prometheus Dashboard for 3.x` dashboard，提供了和 TDinsight 类似的监控 dashboard。 在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。 + + +## taosKeeper 监控指标 + +taosKeeper 也会将自己采集的监控数据写入监控数据库，默认是 `log` 库，可以在 taoskeeper 配置文件中修改。 + +### keeper\_monitor 表 + +`keeper_monitor` 记录 taoskeeper 监控数据。 + +| field | type | is\_tag | comment | +| :------- | :-------- | :------ | :----------- | +| ts | TIMESTAMP | | timestamp | +| cpu | DOUBLE | | cpu 使用率 | +| mem | DOUBLE | | 内存使用率 | +| identify | NCHAR | TAG | 身份标识信息 | diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-1-cluster-status.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-1-cluster-status.webp index 3bc0d960f1..7f7792b2e9 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-1-cluster-status.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-1-cluster-status.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-2-dnodes.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-2-dnodes.webp index f5a602d3f9..9ecc0c5609 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-2-dnodes.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-2-dnodes.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-3-mnodes.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-3-mnodes.webp index f155fa42a0..766d44f620 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-3-mnodes.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-3-mnodes.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-4-requests.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-4-requests.webp index dc0b85e262..00571b4a26 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-4-requests.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-4-requests.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-5-database.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-5-database.webp index 342c8cfc0a..2f7cfe8939 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-5-database.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-5-database.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-6-dnode-usage.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-6-dnode-usage.webp index cc8a912810..826309ba3d 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-6-dnode-usage.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-6-dnode-usage.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-8-taosadapter.webp b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-8-taosadapter.webp index a90477880f..4aec5ce4d0 100644 Binary files a/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-8-taosadapter.webp and b/docs/zh/14-reference/01-components/12-tdinsight/assets/TDinsight-8-taosadapter.webp differ diff --git a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx index 2bceff7e6d..224fd5908d 100644 --- a/docs/zh/14-reference/01-components/12-tdinsight/index.mdx +++ b/docs/zh/14-reference/01-components/12-tdinsight/index.mdx @@ -7,80 +7,47 @@ toc_max_heading_level: 4 import Tabs from '@theme/Tabs' import TabItem from '@theme/TabItem' -TDinsight 是使用监控数据库和 [Grafana] 对 TDengine 进行监控的解决方案。 +TDinsight 是使用 [Grafana] 对 TDengine 进行监控的解决方案。 -TDengine 通过 taosKeeper 将服务器的 CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入指定数据库，并对重要的系统操作（比如登录、创建、删除数据库等）以及各种错误报警信息进行记录。通过 [Grafana] 和 [TDengine 数据源插件](https://github.com/taosdata/grafanaplugin/releases)，TDinsight 将集群状态、节点信息、插入及查询请求、资源使用情况等进行可视化展示，同时还支持 vnode、dnode、mnode 节点状态异常告警，为开发者实时监控 TDengine 集群运行状态提供了便利。本文将指导用户安装 Grafana 服务器并通过 `TDinsight.sh` 安装脚本自动安装 TDengine 数据源插件及部署 TDinsight 可视化面板。 +TDengine 通过 taosKeeper 将服务器的 CPU、内存、硬盘空间、带宽、请求数、磁盘读写速度、慢查询等信息定时写入指定数据库。通过 Grafana 和 TDengine 数据源插件，TDinsight 将集群状态、节点信息、插入及查询请求、资源使用情况等进行可视化展示，为开发者实时监控 TDengine 集群运行状态提供了便利。本文将指导用户安装 TDengine 数据源插件及部署 TDinsight 可视化面板。 -## 系统要求 +## 前置条件 -- 单节点的 TDengine 服务器或多节点的 [TDengine] 集群，以及一个[Grafana]服务器。此仪表盘需要 TDengine 3.0.0.0 及以上，并开启监控服务，具体配置请参考：[TDengine 监控配置](../../../operation/monitor)。 -- taosAdapter 已经安装并正常运行。具体细节请参考：[taosAdapter 使用手册](../taosadapter) -- taosKeeper 已安装并正常运行。注意需要 taos.cfg 文件中打开 monitor 相关配置项，具体细节请参考：[taosKeeper 使用手册](../taoskeeper) +首先检查下面服务： +- TDengine 已经安装并正常运行，此仪表盘需要 TDengine 3.0.0.0 及以上，并开启监控上报配置，具体配置请参考：[TDengine 监控配置](../taosd/#监控相关)。 +- taosAdapter 已经安装并正常运行。具体细节请参考：[taosAdapter 参考手册](../taosadapter) +- taosKeeper 已安装并正常运行。具体细节请参考：[taosKeeper 参考手册](../taoskeeper) -记录以下信息： +然后记录以下信息： -- taosAdapter 集群 REST API 地址，如：`http://tdengine.local:6041`。 +- taosAdapter 集群 REST API 地址，如：`http://localhost:6041`。 - taosAdapter 集群认证信息，可使用用户名及密码。 - taosKeeper 记录监控指标的数据库名称。 -## 安装 Grafana +## 安装和启动 Grafana -我们建议在此处使用最新的[Grafana] 8 或 9 版本。您可以在任何[支持的操作系统](https://grafana.com/docs/grafana/latest/installation/requirements/#supported-operating-systems)中，按照 [Grafana 官方文档安装说明](https://grafana.com/docs/grafana/latest/installation/) 安装 [Grafana]。 +我们建议您使用最新的 Grafana 版本，TDInsight 支持 Grafana 7.5 及以上版本。您可以在任何[支持的操作系统](https://grafana.com/docs/grafana/latest/installation/requirements/#supported-operating-systems)中，按照 [Grafana 官方文档安装说明](https://grafana.com/docs/grafana/latest/installation/) 安装 Grafana。 +安装后请参考 [启动 Grafana](https://grafana.com/docs/grafana/latest/setup-grafana/start-restart-grafana/) 启动 Grafana 服务。 - - +安装完成后就可以在 Web 浏览器中打开 Grafana 网址，默认是：`http://localhost:3000`。 默认用户名/密码都是 `admin`。Grafana 会要求在首次登录后更改密码。 -对于 Debian 或 Ubuntu 操作系统，建议使用 Grafana 镜像仓库。使用如下命令从零开始安装： +:::info -```bash -sudo apt-get install -y apt-transport-https -sudo apt-get install -y software-properties-common wget -wget -q -O - https://packages.grafana.com/gpg.key |\ - sudo apt-key add - -echo "deb https://packages.grafana.com/oss/deb stable main" |\ - sudo tee -a /etc/apt/sources.list.d/grafana.list -sudo apt-get update -sudo apt-get install grafana -``` +下文介绍中，都以 Grafana v11.0.0 版本为例，其他版本功能可能有差异，请参考 [Grafana 官网](https://grafana.com/docs/grafana/latest/)。 - - +::: -您可以从官方 YUM 镜像仓库安装。 +## 安装 TDengine 数据源插件 -```bash -sudo tee /etc/yum.repos.d/grafana.repo << EOF -[grafana] -name=grafana -baseurl=https://packages.grafana.com/oss/rpm -repo_gpgcheck=1 -enabled=1 -gpgcheck=1 -gpgkey=https://packages.grafana.com/gpg.key -sslverify=1 -sslcacert=/etc/pki/tls/certs/ca-bundle.crt -EOF -sudo yum install grafana -``` - -或者用 RPM 安装： - -```bash -wget https://dl.grafana.com/oss/release/grafana-7.5.11-1.x86_64.rpm -sudo yum install grafana-7.5.11-1.x86_64.rpm -# or -sudo yum install \ - https://dl.grafana.com/oss/release/grafana-7.5.11-1.x86_64.rpm -``` - - - - - -### 安装 TDengine 数据源插件 +TDInsight 支持图形界面安装、手动安装和脚本安装三种安装方式，一般建议图形界面安装。对于 Grafana 8.5 以下版本可以使用手动安装和脚本安装方式。 - + + +使用 Grafana 最新版本（8.5+），您可以在 Grafana 中[浏览和管理插件](https://grafana.com/docs/grafana/next/administration/plugin-management/#plugin-catalog)。在 Grafana 管理界面中的 **Configurations > Plugins** 页面直接搜索 `TDengine` 并按照提示安装。 + + + 从 GitHub 安装 TDengine 最新版数据源插件。 @@ -108,9 +75,9 @@ allow_loading_unsigned_plugins = tdengine-datasource - + -我们提供了一个自动化安装脚本 [`TDinsight.sh`](https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh) 脚本以便用户快速进行安装配置。 +我们提供了一个自动化安装脚本 [TDinsight.sh](https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh) 脚本以便用户快速进行安装配置。 您可以通过 `wget` 或其他工具下载该脚本： @@ -122,9 +89,154 @@ chmod +x TDinsight.sh 这个脚本会自动下载最新的[Grafana TDengine 数据源插件](https://github.com/taosdata/grafanaplugin/releases/latest) 和 [TDinsight 仪表盘](https://github.com/taosdata/grafanaplugin/blob/master/dashboards/TDinsightV3.json) ，将命令行选项中的可配置参数转为 [Grafana Provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) 配置文件，以进行自动化部署及更新等操作。 -假设您在同一台主机上使用 TDengine 和 Grafana 的默认服务。运行 `./TDinsight.sh` 并打开 Grafana 浏览器窗口就可以看到 TDinsight 仪表盘了。 +1. 假设您在同一台主机上使用 TDengine 和 Grafana 服务。 运行 `./TDinsight.sh` 并打开 Grafana 页面就可以看到 TDinsight 仪表盘了。 +2. 假设您在主机 `tdengine` 上启动 TDengine 数据库，taosAdapter 的 HTTP 监听端口为 `6041`，用户为 `root1`，密码为 `pass5ord`。执行脚本：`./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord` -下面是 TDinsight.sh 的用法说明： +详细的使用方法请参考 [TDinsight.sh 详细说明](./#附录) + + + + + +## 添加 TDengine 数据源 + +安装完毕后， 点击 “Connections” -> “Data sources“， 然后选择 ”tdengine-datasource“，输入 TDengine 相关配置： +- Host： TDengine 集群中提供 REST 服务的 IP 地址与端口号，默认 `http://localhost:6041` +- User：TDengine 用户名。 +- Password：TDengine 用户密码。 + +点击 `Save & Test` 进行测试，成功会提示：`TDengine Data source is working`。 + + +## 导入 TDengine V3 仪表盘 + +在配置 TDengine 数据源界面，点击 “Dashboards” tab，再点击 ”import” 导入 ”TDengine for 3.x” 仪表盘。 +导入成功后可以进入这个 dashboard，在左上角 ”Log from“ 选项中选择 taosKeeper 中设置的记录监控指标的数据库就可以看到监控结果。 + + +## TDengine V3 仪表盘详情 + +TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态，比如 dnodes、 mnodes、 vnodes 和数据库等。 +主要分为集群状态、DNodes 概述、MNode 概述、请求、数据库、DNode 资源使用情况和 taosAdapter 监控信息。下面我们分别详细介绍。 + +### 集群状态 +这部分指标包括集群当前信息和状态。 + +![TDengine Database TDinsight mnodes overview](./assets/TDinsight-1-cluster-status.webp) + +指标详情（从上到下，从左到右）： + +- **First EP**：当前 TDengine 集群中的`firstEp`设置。 +- **Version**：TDengine 服务器版本（master mnode）。 +- **Expire Time** - 企业版过期时间。 +- **Used Measuring Points** - 企业版已使用的测点数。 +- **Databases** - 数据库个数。 +- **Connections** - 当前连接个数。 +- **DNodes/MNodes/VGroups/VNodes**：每种资源的总数和存活数。 +- **DNodes/MNodes/VGroups/VNodes Alive Percent**：每种资源的存活数/总数的比例，启用告警规则，并在资源存活率（1 分钟内平均健康资源比例）不足 100%时触发。 +- **Measuring Points Used**：启用告警规则的测点数用量（社区版无数据，默认情况下是健康的）。 + +### DNodes 概述 +这部分指标包括集群 dnode 基本信息。 + +![TDengine Database TDinsight mnodes overview](./assets/TDinsight-2-dnodes.webp) + +指标详情： +- **DNodes Status**：`show dnodes` 的简单表格视图。 +- **DNodes Number**：DNodes 数量变化。 + +### MNode 概述 +这部分指标包括集群 mnode 基本信息。 + +![TDengine Database TDinsight mnodes overview](./assets/TDinsight-3-mnodes.webp) + +指标详情： +1. **MNodes Status**：`show mnodes` 的简单表格视图。 +2. **MNodes Number**：类似于`DNodes Number`，MNodes 数量变化。 + +### 请求统计 +这部分指标包括集群执行 sql 的统计指标。 + +![TDengine Database TDinsight requests](./assets/TDinsight-4-requests.webp) + +指标详情： +1. **Select Request**：select 请求数。 +2. **Delete Request**：delete 请求数。 +3. **Insert Request**：insert 请求数。 +4. **Inserted Rows**：实际插入行数。 +5. **Slow Sql**：慢查询数，可以在顶部分时长段过滤。 + +### 表统计 + +这部分指标包括集群中表的统计指标。 + +![TDengine Database TDinsight database](./assets/TDinsight-5-database.webp) + +指标详情： +1. **STables**：超级表数量。 +2. **Total Tables**：所有表数量。 +3. **Tables**：所有普通表数量随时间变化图。 +4. **Tables Number Foreach VGroups**：每个 VGroups 包含的表数量。 + +### DNode 资源使用情况 + +这部分指标包括集群所有数据节点资源使用情况展示，每个数据节点为一个 Row 进行展示。。 + +![TDengine Database TDinsight dnode-usage](./assets/TDinsight-6-dnode-usage.webp) + +指标详情（从上到下，从左到右）： + +1. **Uptime**：从创建 dnode 开始经过的时间。 +2. **Has MNodes?**：当前 dnode 是否为 mnode。 +3. **CPU Cores**：CPU 核数。 +4. **VNodes Number**：当前 dnode 的 VNodes 数量。 +5. **VNodes Masters**：处于 master 角色的 vnode 数量。 +6. **Current CPU Usage of taosd**：taosd 进程的 CPU 使用率。 +7. **Current Memory Usage of taosd**：taosd 进程的内存使用情况。 +8. **Max Disk Used**：taosd 所有数据目录对应的最大磁盘使用率。 +9. **CPU Usage**：进程和系统 CPU 使用率。 +10. **RAM Usage**：RAM 使用指标时间序列视图。 +11. **Disk Used**：多级存储下每个级别使用的磁盘（默认为 level0 级）。 +12. **Disk IO**：磁盘 IO 速率。 +13. **Net IO**：网络 IO，除本机网络之外的总合网络 IO 速率。 + + +### taosAdapter 监控 + +这部分指标包括 taosAdapter rest 和 websocket 请求统计详情。 + +![TDengine Database TDinsight monitor taosadapter](./assets/TDinsight-8-taosadapter.webp) + +指标详情： + +1. **Total**：总请求数 +2. **Successful**：总成功数 +3. **Failed**：总失败数 +4. **Queries**：总查询数 +5. **Writes**：总写入数 +6. **Other**：总其他请求数 + +还有上述分类的细分维度折线图。 + +## 升级 +下面三种方式都可以进行升级： +- 用图形界面，若有新版本，可以在 ”TDengine Datasource“ 插件页面点击 update 升级。 +- 按照手动安装步骤自行安装新的 Grafana 插件和 Dashboard。 +- 通过重新运行 `TDinsight.sh` 脚本升级到最新的 Grafana 插件和 TDinsight Dashboard。 + +## 卸载 +针对不同的安装方式，卸载时： +- 用图形界面，在 ”TDengine Datasource“ 插件页面点击 ”Uninstall“ 卸载。 +- 通过 `TDinsight.sh` 脚本安装的 TDinsight，可以使用命令行 `TDinsight.sh -R` 清理相关资源。 +- 手动安装的 TDinsight，要完全卸载，需要清理以下内容： + 1. Grafana 中的 TDinsight Dashboard。 + 2. Grafana 中的 Data Source 数据源。 + 3. 从插件安装目录删除 `tdengine-datasource` 插件。 + +## 附录 + +### TDinsight.sh 详细说明 +下面是 TDinsight.sh 的用法详细说明： ```text Usage: @@ -189,211 +301,3 @@ sudo ./TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata - 请注意，配置数据源、通知 Channel 和仪表盘在前端是不可更改的。您应该再次通过此脚本更新配置或手动更改 `/etc/grafana/provisioning` 目录（这是 Grafana 的默认目录，根据需要使用`-P`选项更改）中的配置文件。 特别地，当您使用 Grafana Cloud 或其他组织时，`-O` 可用于设置组织 ID。 `-G` 可指定 Grafana 插件安装目录。 `-e` 参数将仪表盘设置为可编辑。 - - - - -### 启动 Grafana 服务 - -```bash -sudo systemctl start grafana-server -sudo systemctl enable grafana-server -``` - -### 登录到 Grafana - -在 Web 浏览器中打开默认的 Grafana 网址：`http://localhost:3000`。 -默认用户名/密码都是 `admin`。Grafana 会要求在首次登录后更改密码。 - -### 添加 TDengine 数据源 - -指向 **Configurations** -> **Data Sources** 菜单，然后点击 **Add data source** 按钮。 - -![TDengine Database TDinsight 添加数据源按钮](./assets/howto-add-datasource-button.webp) - -搜索并选择**TDengine**。 - -![TDengine Database TDinsight 添加数据源](./assets/howto-add-datasource-tdengine.webp) - -配置 TDengine 数据源。 - -![TDengine Database TDinsight 数据源配置](./assets/howto-add-datasource.webp) - -保存并测试，正常情况下会报告 'TDengine Data source is working'。 - -![TDengine Database TDinsight 数据源测试](./assets/howto-add-datasource-test.webp) - - - -### 导入仪表盘 - -在配置 TDengine 数据源界面，点击 **Dashboards** tab。 - -![TDengine Database TDinsight 导入仪表盘和配置](./assets/import_dashboard.webp) - -选择 `TDengine for 3.x`，并点击 `import`。 - -导入完成后，在搜索界面已经出现了 **TDinsight for 3.x** dashboard。 - -![TDengine Database TDinsight 查看导入结果](./assets/import_dashboard_view.webp) - -进入 TDinsight for 3.x dashboard 后，选择 taosKeeper 中设置的记录监控指标的数据库。 - -![TDengine Database TDinsight 选择数据库](./assets/select_dashboard_db.webp) - -然后可以看到监控结果。 - -## TDinsight 仪表盘详细信息 - -TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态，比如 dnodes、 mnodes、 vnodes 和数据库等。 - -指标详情如下： - -### 集群状态 - -![TDengine Database TDinsight mnodes overview](./assets/TDinsight-1-cluster-status.webp) - -这部分包括集群当前信息和状态，告警信息也在此处（从左到右，从上到下）。 - -- **First EP**：当前 TDengine 集群中的`firstEp`设置。 -- **Version**：TDengine 服务器版本（master mnode）。 -- **Master Uptime**: 当前 Master MNode 被选举为 Master 后经过的时间。 -- **Expire Time** - 企业版过期时间。 -- **Used Measuring Points** - 企业版已使用的测点数。 -- **Databases** - 数据库个数。 -- **Connections** - 当前连接个数。 -- **DNodes/MNodes/VGroups/VNodes**：每种资源的总数和存活数。 -- **DNodes/MNodes/VGroups/VNodes Alive Percent**：每种资源的存活数/总数的比例，启用告警规则，并在资源存活率（1 分钟内平均健康资源比例）不足 100%时触发。 -- **Measuring Points Used**：启用告警规则的测点数用量（社区版无数据，默认情况下是健康的）。 -- **Grants Expire Time**：启用告警规则的企业版过期时间（社区版无数据，默认情况是健康的）。 -- **Error Rate**：启用警报的集群总合错误率（每秒平均错误数）。 - -### DNodes 状态 - -![TDengine Database TDinsight mnodes overview](./assets/TDinsight-2-dnodes.webp) - -- **DNodes Status**：`show dnodes` 的简单表格视图。 -- **DNodes Lifetime**：从创建 dnode 开始经过的时间。 -- **DNodes Number**：DNodes 数量变化。 - -### MNode 概述 - -![TDengine Database TDinsight mnodes overview](./assets/TDinsight-3-mnodes.webp) - -1. **MNodes Status**：`show mnodes` 的简单表格视图。 -2. **MNodes Number**：类似于`DNodes Number`，MNodes 数量变化。 - -### 请求 - -![TDengine Database TDinsight requests](./assets/TDinsight-4-requests.webp) - -1. **Requests Rate(Inserts per Second)**：平均每秒插入次数。 -2. **Requests (Selects)**：查询请求数及变化率（count of second）。 - -### 数据库 - -![TDengine Database TDinsight database](./assets/TDinsight-5-database.webp) - -数据库使用情况，对变量 `$database` 的每个值即每个数据库进行重复多行展示。 - -1. **STables**：超级表数量。 -2. **Total Tables**：所有表数量。 -3. **Tables**：所有普通表数量随时间变化图。 -4. **Tables Number Foreach VGroups**：每个 VGroups 包含的表数量。 - -### DNode 资源使用情况 - -![TDengine Database TDinsight dnode-usage](./assets/TDinsight-6-dnode-usage.webp) - -数据节点资源使用情况展示，对变量 `$fqdn` 即每个数据节点进行重复多行展示。包括： - -1. **Uptime**：从创建 dnode 开始经过的时间。 -2. **Has MNodes?**：当前 dnode 是否为 mnode。 -3. **CPU Cores**：CPU 核数。 -4. **VNodes Number**：当前 dnode 的 VNodes 数量。 -5. **VNodes Masters**：处于 master 角色的 vnode 数量。 -6. **Current CPU Usage of taosd**：taosd 进程的 CPU 使用率。 -7. **Current Memory Usage of taosd**：taosd 进程的内存使用情况。 -8. **Disk Used**：taosd 数据目录的总磁盘使用百分比。 -9. **CPU Usage**：进程和系统 CPU 使用率。 -10. **RAM Usage**：RAM 使用指标时间序列视图。 -11. **Disk Used**：多级存储下每个级别使用的磁盘（默认为 level0 级）。 -12. **Disk Increasing Rate per Minute**：每分钟磁盘用量增加或减少的百分比。 -13. **Disk IO**：磁盘 IO 速率。 -14. **Net IO**：网络 IO，除本机网络之外的总合网络 IO 速率。 - -### 登录历史 - -![TDengine Database TDinsight 登录历史](./assets/TDinsight-7-login-history.webp) - -目前只报告每分钟登录次数。 - -### 监控 taosAdapter - -![TDengine Database TDinsight monitor taosadapter](./assets/TDinsight-8-taosadapter.webp) - -支持监控 taosAdapter 请求统计和状态详情。包括： - -1. **Http Request Total**: 请求总数。 -2. **Http Request Fail**: 请求总数。 -3. **CPU Used**: taosAdapter CPU 使用情况。 -4. **Memory Used**: taosAdapter 内存使用情况。 -5. **Http Request Inflight**: 即时处理请求数。 -6. **Http Status Code**: taosAdapter http 状态码。 - -## 升级 - -通过 `TDinsight.sh` 脚本安装的 TDinsight，可以通过重新运行该脚本就可以升级到最新的 Grafana 插件和 TDinsight Dashboard。 - -手动安装的情况下，可按照上述步骤自行安装新的 Grafana 插件和 Dashboard。 - -## 卸载 - -通过 `TDinsight.sh` 脚本安装的 TDinsight，可以使用命令行 `TDinsight.sh -R` 清理相关资源。 - -手动安装时，要完全卸载 TDinsight，需要清理以下内容： - -1. Grafana 中的 TDinsight Dashboard。 -2. Grafana 中的 Data Source 数据源。 -3. 从插件安装目录删除 `tdengine-datasource` 插件。 - -## 整合的 Docker 示例 - -```bash -git clone --depth 1 https://github.com/taosdata/grafanaplugin.git -cd grafanaplugin -``` - -根据需要在 `docker-compose.yml` 文件中修改： - -```yaml -version: '3.7' - -services: - grafana: - image: grafana/grafana:7.5.10 - volumes: - - ./dist:/var/lib/grafana/plugins/tdengine-datasource - - ./grafana/grafana.ini:/etc/grafana/grafana.ini - - ./grafana/provisioning/:/etc/grafana/provisioning/ - - grafana-data:/var/lib/grafana - environment: - TDENGINE_API: ${TDENGINE_API} - TDENGINE_USER: ${TDENGINE_USER} - TDENGINE_PASS: ${TDENGINE_PASS} - ports: - - 3000:3000 -volumes: - grafana-data: -``` - -替换`docker-compose.yml`中的环境变量或保存环境变量到`.env`文件，然后用`docker-compose up`启动 Grafana。`docker-compose` 工具的具体用法参见 [Docker Compose Reference](https://docs.docker.com/compose/) - -```bash -docker-compose up -d -``` - -TDinsight 已经通过 Provisioning 部署完毕，请到 http://localhost:3000/d/tdinsight/ 查看仪表盘。 - -[grafana]: https://grafana.com -[tdengine]: https://www.taosdata.com diff --git a/docs/zh/14-reference/05-connector/26-rust.mdx b/docs/zh/14-reference/05-connector/26-rust.mdx index dcaf0cf219..6b0ec4a68d 100644 --- a/docs/zh/14-reference/05-connector/26-rust.mdx +++ b/docs/zh/14-reference/05-connector/26-rust.mdx @@ -35,6 +35,7 @@ Websocket 连接支持所有能运行 Rust 的平台。 | Rust 连接器版本 | TDengine 版本 | 主要功能 | | :----------------: | :--------------: | :--------------------------------------------------: | +| v0.12.3 | 3.3.0.0 or later | 优化了 Websocket 查询和插入性能，支持了 VARBINARY 和 GEOMETRY 类型 | | v0.12.0 | 3.2.3.0 or later | WS 支持压缩。 | | v0.11.0 | 3.2.0.0 | TMQ 功能优化。 | | v0.10.0 | 3.1.0.0 | WS endpoint 变更。 | @@ -44,7 +45,6 @@ Websocket 连接支持所有能运行 Rust 的平台。 | v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 | | v0.6.0 | 3.0.0.0 | 基础功能。 | -Rust 连接器仍然在快速开发中，1.0 之前无法保证其向后兼容。建议使用 3.0 版本以上的 TDengine，以避免已知问题。 ## 处理错误 diff --git a/docs/zh/14-reference/05-connector/30-python.mdx b/docs/zh/14-reference/05-connector/30-python.mdx index 714848a2ba..671234fb7f 100644 --- a/docs/zh/14-reference/05-connector/30-python.mdx +++ b/docs/zh/14-reference/05-connector/30-python.mdx @@ -41,12 +41,16 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con |Python Connector 版本|主要变化| |:-------------------:|:----:| +|2.7.15|新增 VARBINARY 和 GEOMETRY 类型支持| +|2.7.14|修复已知问题| +|2.7.13|新增 tmq 同步提交 offset 接口| |2.7.12|1. 新增 varbinary 类型支持（STMT暂不支持 varbinary ）
2. query 性能提升（感谢贡献者[hadrianl](https://github.com/taosdata/taos-connector-python/pull/209)）| |2.7.9|数据订阅支持获取消费进度和重置消费进度| |2.7.8|新增 `execute_many`| |Python Websocket Connector 版本|主要变化| |:----------------------------:|:-----:| +|0.3.2|优化 Websocket sql 查询和插入性能，修改 readme 和 文档，修复已知问题| |0.2.9|已知问题修复| |0.2.5|1. 数据订阅支持获取消费进度和重置消费进度
2. 支持 schemaless
3. 支持 STMT| |0.2.4|数据订阅新增取消订阅方法| diff --git a/docs/zh/14-reference/05-connector/index.md b/docs/zh/14-reference/05-connector/index.md index 8356e89c56..1c051bf163 100644 --- a/docs/zh/14-reference/05-connector/index.md +++ b/docs/zh/14-reference/05-connector/index.md @@ -28,13 +28,14 @@ TDengine 提供了丰富的应用程序开发接口，为了便于用户快速 TDengine 版本更新往往会增加新的功能特性，列表中的连接器版本为连接器最佳适配版本。 -| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| ---------------------- | --------- | ---------- | ------------ | ------------- | --------------- | -------- | -| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | -| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | -| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | -| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | +| **TDengine 版本** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| ---------------------- | ------------- | ------------------------------------------- | ------------ | ------------- | --------------- | -------- | +| **3.3.0.0 及以上** | 3.3.2.0及以上 | taospy 2.7.15及以上，taos-ws-py 0.3.2及以上 | 3.5.5及以上 | 3.1.3及以上 | 3.1.0及以上 | 当前版本 | +| **3.0.0.0 及以上** | 3.0.2以上 | 当前版本 | 3.0 分支 | 3.0.0 | 3.1.0 | 当前版本 | +| **2.4.0.14 及以上** | 2.0.38 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | +| **2.4.0.4 - 2.4.0.13** | 2.0.37 | 当前版本 | develop 分支 | 1.0.2 - 1.0.6 | 2.0.10 - 2.0.12 | 当前版本 | +| **2.2.x.x ** | 2.0.36 | 当前版本 | master 分支 | n/a | 2.0.7 - 2.0.9 | 当前版本 | +| **2.0.x.x ** | 2.0.34 | 当前版本 | master 分支 | n/a | 2.0.1 - 2.0.6 | 当前版本 | ## 功能特性 diff --git a/include/client/taos.h b/include/client/taos.h index 1d2b3a913c..73ab52357a 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -270,7 +270,10 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int int precision, int32_t ttl); DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid); - +DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, + int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey); +DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, + int precision, int32_t ttl, int64_t reqid, char *tbnameKey); /* --------------------------TMQ INTERFACE------------------------------- */ typedef struct tmq_t tmq_t; diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index a0ff3353bc..61ae034450 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -245,12 +245,12 @@ typedef struct SStoreSnapshotFn { } SStoreSnapshotFn; typedef struct SStoreMeta { - SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor - void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor - void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor - void (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first, int8_t move); // metaResumeTbCursor - int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext - int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev + SMTbCursor* (*openTableMetaCursor)(void* pVnode); // metaOpenTbCursor + void (*closeTableMetaCursor)(SMTbCursor* pTbCur); // metaCloseTbCursor + void (*pauseTableMetaCursor)(SMTbCursor* pTbCur); // metaPauseTbCursor + int32_t (*resumeTableMetaCursor)(SMTbCursor* pTbCur, int8_t first, int8_t move); // metaResumeTbCursor + int32_t (*cursorNext)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorNext + int32_t (*cursorPrev)(SMTbCursor* pTbCur, ETableType jumpTableType); // metaTbCursorPrev int32_t (*getTableTags)(void* pVnode, uint64_t suid, SArray* uidList); int32_t (*getTableTagsByUid)(void* pVnode, int64_t suid, SArray* uidList); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 04b92a897a..0b59d7c2f5 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -320,6 +320,30 @@ typedef int32_t (*TScriptUdfDestoryFunc)(void *udfCtx); typedef int32_t (*TScriptOpenFunc)(SScriptUdfEnvItem *items, int numItems); typedef int32_t (*TScriptCloseFunc)(); +// clang-format off +#ifdef WINDOWS + #define fnFatal(...) {} + #define fnError(...) {} + #define fnWarn(...) {} + #define fnInfo(...) {} + #define fnDebug(...) {} + #define fnTrace(...) {} +#else + DLL_EXPORT void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 4, 5))) +#endif + ; + extern int32_t udfDebugFlag; + #define udfFatal(...) { if (udfDebugFlag & 1) { taosPrintLog("UDF FATAL ", 1, 255, __VA_ARGS__); }} + #define udfError(...) { if (udfDebugFlag & 1) { taosPrintLog("UDF ERROR ", 1, 255, __VA_ARGS__); }} + #define udfWarn(...) { if (udfDebugFlag & 2) { taosPrintLog("UDF WARN ", 2, 255, __VA_ARGS__); }} + #define udfInfo(...) { if (udfDebugFlag & 2) { taosPrintLog("UDF ", 2, 255, __VA_ARGS__); }} + #define udfDebug(...) { if (udfDebugFlag & 4) { taosPrintLog("UDF ", 4, udfDebugFlag, __VA_ARGS__); }} + #define udfTrace(...) { if (udfDebugFlag & 8) { taosPrintLog("UDF ", 8, udfDebugFlag, __VA_ARGS__); }} +#endif +// clang-format on + #ifdef __cplusplus } #endif diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f56860dd4f..6e2b83dce7 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -25,9 +25,9 @@ extern "C" { #include "tarray.h" #include "thash.h" #include "tlog.h" -#include "tsimplehash.h" #include "tmsg.h" #include "tmsgcb.h" +#include "tsimplehash.h" typedef enum { JOB_TASK_STATUS_NULL = 0, @@ -69,16 +69,16 @@ typedef enum { #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) #define QUERY_MSG_MASK_AUDIT() (1 << 1) #define QUERY_MSG_MASK_VIEW() (1 << 2) -#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) -#define TEST_AUDIT_MASK(m) (((m) & QUERY_MSG_MASK_AUDIT()) != 0) -#define TEST_VIEW_MASK(m) (((m) & QUERY_MSG_MASK_VIEW()) != 0) +#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) +#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema uint8_t precision; // the number of precision col_id_t numOfColumns; // the number of columns int16_t numOfPKs; - int32_t rowSize; // row size of the schema + int32_t rowSize; // row size of the schema } STableComInfo; typedef struct SIndexMeta { @@ -119,8 +119,9 @@ typedef struct STableMeta { int32_t sversion; int32_t tversion; STableComInfo tableInfo; - SSchemaExt* schemaExt; // There is no additional memory allocation, and the pointer is fixed to the next address of the schema content. - SSchema schema[]; + SSchemaExt* schemaExt; // There is no additional memory allocation, and the pointer is fixed to the next address of + // the schema content. + SSchema schema[]; } STableMeta; #pragma pack(pop) @@ -196,9 +197,9 @@ typedef struct SBoundColInfo { } SBoundColInfo; typedef struct STableColsData { - char tbName[TSDB_TABLE_NAME_LEN]; - SArray* aCol; - bool getFromHash; + char tbName[TSDB_TABLE_NAME_LEN]; + SArray* aCol; + bool getFromHash; } STableColsData; typedef struct STableVgUid { @@ -207,15 +208,14 @@ typedef struct STableVgUid { } STableVgUid; typedef struct STableBufInfo { - void* pCurBuff; - SArray* pBufList; - int64_t buffUnit; - int64_t buffSize; - int64_t buffIdx; - int64_t buffOffset; + void* pCurBuff; + SArray* pBufList; + int64_t buffUnit; + int64_t buffSize; + int64_t buffIdx; + int64_t buffOffset; } STableBufInfo; - typedef struct STableDataCxt { STableMeta* pMeta; STSchema* pSchema; @@ -237,23 +237,22 @@ typedef struct SStbInterlaceInfo { void* pRequest; uint64_t requestId; int64_t requestSelf; - bool tbFromHash; + bool tbFromHash; SHashObj* pVgroupHash; SArray* pVgroupList; SSHashObj* pTableHash; int64_t tbRemainNum; STableBufInfo tbBuf; char firstName[TSDB_TABLE_NAME_LEN]; - STSchema *pTSchema; - STableDataCxt *pDataCtx; - void *boundTags; + STSchema* pTSchema; + STableDataCxt* pDataCtx; + void* boundTags; - bool tableColsReady; - SArray *pTableCols; - int32_t pTableColsIdx; + bool tableColsReady; + SArray* pTableCols; + int32_t pTableColsIdx; } SStbInterlaceInfo; - typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -308,6 +307,8 @@ void destroyAhandle(void* ahandle); int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo, bool persistHandle, void* ctx); +int32_t asyncFreeConnById(void* pTransporter, int64_t pid); +; /** * Asynchronously send message to server, after the response received, the callback will be incured. * @@ -325,7 +326,7 @@ void initQueryModuleMsgHandle(); const SSchema* tGetTbnameColumnSchema(); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); -int32_t getAsofJoinReverseOp(EOperatorType op); +int32_t getAsofJoinReverseOp(EOperatorType op); int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); @@ -384,7 +385,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ - (_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW || \ + (_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW || \ (_type) == TDMT_MND_CREATE_TSMA || (_type) == TDMT_MND_DROP_TSMA || (_type) == TDMT_MND_DROP_TB_WITH_TSMA) #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 6c0d04354a..5b860cc23a 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -125,6 +125,7 @@ typedef struct SRpcInit { int32_t timeToGetConn; int8_t supportBatch; // 0: no batch, 1. batch int32_t batchSize; + int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait void *parent; } SRpcInit; @@ -158,18 +159,21 @@ void *rpcReallocCont(void *ptr, int64_t contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side // Please use tmsg functions, which are defined in tmsgcb.h -int rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); -int rpcSendResponse(const SRpcMsg *pMsg); -int rpcRegisterBrokenLinkArg(SRpcMsg *msg); -int rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock +int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +int32_t rpcSendResponse(const SRpcMsg *pMsg); +int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); +int32_t rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock // These functions will not be called in the child process -int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); -int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); -int rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, +int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +int32_t rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int32_t rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, int32_t timeoutMs); -int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); -void *rpcAllocHandle(); + +int32_t rpcFreeConnById(void *shandle, int64_t connId); + +int32_t rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); +int32_t rpcAllocHandle(int64_t *refId); int32_t rpcSetIpWhite(void *thandl, void *arg); int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); diff --git a/include/util/tcompare.h b/include/util/tcompare.h index 80f992f646..c7a29cad57 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -49,6 +49,7 @@ int32_t InitRegexCache(); void DestroyRegexCache(); int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo); int32_t checkRegexPattern(const char *pPattern); +void DestoryThreadLocalRegComp(); int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo); diff --git a/include/util/tlog.h b/include/util/tlog.h index 67aafbfe44..b4dbbef532 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -74,13 +74,13 @@ void taosCloseLog(); void taosResetLog(); void taosDumpData(uint8_t *msg, int32_t len); -void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) +void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ __attribute__((format(printf, 4, 5))) #endif ; -void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) +void taosPrintLongString(const char *flags, int32_t level, int32_t dflag, const char *format, ...) #ifdef __GNUC__ __attribute__((format(printf, 4, 5))) #endif diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 25f3d8ce4a..d975841e83 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -18,6 +18,7 @@ cfg_link_dir="/usr/local/taos/cfg" service_config_dir="/etc/systemd/system" taos_service_name="taosd" taoskeeper_service_name="taoskeeper" +explorer_service_name="taos-explorer" csudo="" if command -v sudo > /dev/null; then csudo="sudo " @@ -64,6 +65,13 @@ function kill_taoskeeper() { fi } +function kill_taos-explorer() { + pid=$(ps -ef | grep "taos-explorer" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function clean_service_on_systemd() { taosadapter_service_config="${service_config_dir}/taosadapter.service" if systemctl is-active --quiet taosadapter; then @@ -89,6 +97,13 @@ function clean_service_on_systemd() { ${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null fi [ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config} + + explorer_service_config="${service_config_dir}/${explorer_service_name}.service" + if systemctl is-active --quiet ${explorer_service_name}; then + echo "TDengine taoskeeper is running, stopping it..." + ${csudo}systemctl stop ${explorer_service_name} &> /dev/null || echo &> /dev/null + fi + [ -f ${explorer_service_config} ] && ${csudo}rm -f ${explorer_service_config} } function clean_service_on_sysvinit() { @@ -125,6 +140,7 @@ function clean_service() { kill_taosadapter kill_taosd kill_taoskeeper + kill_taos-explorer fi } @@ -137,16 +153,23 @@ ${csudo}rm -f ${bin_link_dir}/taosd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : +${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/taoskeeper || : +${csudo}rm -f ${bin_link_dir}/taos-explorer || : +${csudo}rm -f ${bin_link_dir}/start-all.sh || : +${csudo}rm -f ${bin_link_dir}/stop-all.sh || : ${csudo}rm -f ${cfg_link_dir}/*.new || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/tdef.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : +${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : +${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : +${csudo}rm -f ${lib64_link_dir}/libtaosws.so || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h index e901f86b41..209c376f30 100644 --- a/source/client/inc/clientSml.h +++ b/source/client/inc/clientSml.h @@ -204,6 +204,7 @@ typedef struct { STableMeta *currSTableMeta; STableDataCxt *currTableDataCtx; bool needModifySchema; + char *tbnameKey; } SSmlHandle; extern int64_t smlFactorNS[]; @@ -219,9 +220,10 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); -int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); + +int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo); int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta); -int32_t smlSetCTableName(SSmlTableInfo *oneTable); +int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey); int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta); int32_t is_same_child_table_telnet(const void *a, const void *b); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 35e6651c41..d63701c529 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -374,8 +374,8 @@ int32_t openTransporter(const char *user, const char *auth, int32_t numOfThread, *pDnodeConn = rpcOpen(&rpcInit); if (*pDnodeConn == NULL) { - tscError("failed to init connection to server."); - code = TSDB_CODE_FAILED; + tscError("failed to init connection to server since %s", tstrerror(terrno)); + code = terrno; } return code; @@ -526,19 +526,17 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj int32_t code = TSDB_CODE_SUCCESS; *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == *pRequest) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } STscObj *pTscObj = acquireTscObj(connId); if (pTscObj == NULL) { - code = TSDB_CODE_TSC_DISCONNECTED; - goto _return; + TSC_ERR_JRET(TSDB_CODE_TSC_DISCONNECTED); } SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); if (interParam == NULL) { releaseTscObj(connId); - code = TSDB_CODE_OUT_OF_MEMORY; - goto _return; + TSC_ERR_JRET(terrno); } TSC_ERR_JRET(tsem_init(&interParam->sem, 0, 0)); interParam->pRequest = *pRequest; @@ -566,7 +564,11 @@ int32_t createRequest(uint64_t connId, int32_t type, int64_t reqid, SRequestObj return TSDB_CODE_SUCCESS; _return: - doDestroyRequest(*pRequest); + if ((*pRequest)->pTscObj) { + doDestroyRequest(*pRequest); + } else { + taosMemoryFree(*pRequest); + } return code; } @@ -869,7 +871,7 @@ _return: TSC_ERR_RET(terrno); } - return TSDB_CODE_SUCCESS; + return code; } void tscStopCrashReport() { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index a174ae4b13..415c2d6685 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1405,7 +1405,7 @@ _return: TSC_ERR_RET(terrno); } - return TSDB_CODE_SUCCESS; + return code; } static void hbStopThread() { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 4aa78caa15..664de5619f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2467,7 +2467,8 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de clientRpc = rpcOpen(&rpcInit); if (clientRpc == NULL) { - tscError("failed to init server status client"); + code = terrno; + tscError("failed to init server status client since %s", tstrerror(code)); goto _OVER; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index c244d36eb7..55be514e40 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -402,7 +402,7 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) { if (kv->valueEscaped) kv->value = NULL; } - code = smlSetCTableName(tinfo); + code = smlSetCTableName(tinfo, info->tbnameKey); if (code != TSDB_CODE_SUCCESS){ smlDestroyTableInfo(&tinfo); return code; @@ -486,10 +486,10 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) return TSDB_CODE_SUCCESS; } -static int32_t smlParseTableName(SArray *tags, char *childTableName) { +static int32_t smlParseTableName(SArray *tags, char *childTableName, char *tbnameKey) { bool autoChildName = false; size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); - if (delimiter > 0) { + if(delimiter > 0 && tbnameKey == NULL){ size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); for (int i = 0; i < taosArrayGetSize(tags); i++) { SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); @@ -517,8 +517,11 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { if (tsSmlDot2Underline) { smlStrReplace(childTableName, strlen(childTableName)); } - } else { - size_t childTableNameLen = strlen(tsSmlChildTableName); + }else{ + if (tbnameKey == NULL){ + tbnameKey = tsSmlChildTableName; + } + size_t childTableNameLen = strlen(tbnameKey); if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; for (int i = 0; i < taosArrayGetSize(tags); i++) { @@ -527,7 +530,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { return TSDB_CODE_SML_INVALID_DATA; } // handle child table name - if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tbnameKey, tag->keyLen) == 0) { (void)memset(childTableName, 0, TSDB_TABLE_NAME_LEN); (void)strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); if (tsSmlDot2Underline) { @@ -542,8 +545,8 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { return TSDB_CODE_SUCCESS; } -int32_t smlSetCTableName(SSmlTableInfo *oneTable) { - int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName); +int32_t smlSetCTableName(SSmlTableInfo *oneTable, char *tbnameKey) { + int32_t code = smlParseTableName(oneTable->tags, oneTable->childTableName, tbnameKey); if(code != TSDB_CODE_SUCCESS){ return code; } @@ -2127,7 +2130,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL } TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, - int protocol, int precision, int32_t ttl, int64_t reqid) { + int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey) { int32_t code = TSDB_CODE_SUCCESS; if (NULL == taos) { uError("SML:taos_schemaless_insert error taos is null"); @@ -2159,6 +2162,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->lineNum = numLines; + info->tbnameKey = tbnameKey; smlSetReqSQL(request, lines, rawLine, rawLineEnd); @@ -2237,9 +2241,14 @@ end: * @return TAOS_RES */ +TAOS_RES *taos_schemaless_insert_ttl_with_reqid_tbname_key(TAOS *taos, char *lines[], int numLines, int protocol, + int precision, int32_t ttl, int64_t reqid, char *tbnameKey){ + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid, tbnameKey); +} + TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); + return taos_schemaless_insert_ttl_with_reqid_tbname_key(taos, lines, numLines, protocol, precision, ttl, reqid, NULL); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { @@ -2272,10 +2281,15 @@ static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol } } +TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(TAOS *taos, char *lines, int len, int32_t *totalRows, + int protocol, int precision, int32_t ttl, int64_t reqid, char *tbnameKey){ + getRawLineLen(lines, len, totalRows, protocol); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid, tbnameKey); +} + TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl, int64_t reqid) { - getRawLineLen(lines, len, totalRows, protocol); - return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); + return taos_schemaless_insert_raw_ttl_with_reqid_tbname_key(taos, lines, len, totalRows, protocol, precision, ttl, reqid, NULL); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0a3c365c59..197a65add8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -511,7 +511,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; + SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); if(tEncodeMqVgOffset(&encoder, &pOffset) < 0) { tEncoderClear(&encoder); @@ -953,7 +953,7 @@ void tmqSendHbReq(void* param, void* tmrId) { tscError("tmqSendHbReq asyncSendMsgToServer failed"); } - atomic_val_compare_exchange_8(&pollFlag, 1, 0); + (void)atomic_val_compare_exchange_8(&pollFlag, 1, 0); OVER: tDestroySMqHbReq(&req); if(tmrId != NULL){ @@ -1275,7 +1275,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { // init semaphore if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) { - tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), + tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, tstrerror(TAOS_SYSTEM_ERROR(errno)), pTmq->groupId); SET_ERROR_MSG_TMQ("init t_sem failed") goto _failed; @@ -2394,7 +2394,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } } - atomic_val_compare_exchange_8(&pollFlag, 0, 1); + (void)atomic_val_compare_exchange_8(&pollFlag, 0, 1); while (1) { tmqHandleAllDelayedTask(tmq); @@ -3133,7 +3133,7 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; + SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); code = tEncodeMqVgOffset(&encoder, &pOffset); if (code < 0) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b489314e21..f8ae16a05c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1261,13 +1261,16 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } -static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { +static int32_t createHelpColInfoData(const SSDataBlock* pDataBlock, SColumnInfoData** ppCols) { + *ppCols = NULL; + + int32_t code = 0; int32_t rows = pDataBlock->info.capacity; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { - return NULL; + return terrno; } for (int32_t i = 0; i < numOfCols; ++i) { @@ -1280,8 +1283,11 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t)); pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length); - if (pCols[i].varmeta.offset == NULL) { - return NULL; + if (pCols[i].varmeta.offset == NULL || pCols[i].pData == NULL) { + code = terrno; + taosMemoryFree(pCols[i].varmeta.offset); + taosMemoryFree(pCols[i].pData); + goto _error; } pCols[i].varmeta.length = pColInfoData->varmeta.length; @@ -1290,12 +1296,20 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { pCols[i].nullbitmap = taosMemoryCalloc(1, BitmapLen(rows)); pCols[i].pData = taosMemoryCalloc(rows, pCols[i].info.bytes); if (pCols[i].nullbitmap == NULL || pCols[i].pData == NULL) { - return NULL; + code = terrno; + taosMemoryFree(pCols[i].nullbitmap); + taosMemoryFree(pCols[i].pData); + goto _error; } } } - return pCols; + *ppCols = pCols; + return code; + + _error: + taosMemoryFree(pCols); + return code; } static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { @@ -1423,16 +1437,15 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { int64_t p1 = taosGetTimestampUs(); - SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); - if (pCols == NULL) { + SColumnInfoData* pCols = NULL; + int32_t code = createHelpColInfoData(pDataBlock, &pCols); + if (code != 0) { destroyTupleIndex(index); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return terrno; + return code; } int64_t p2 = taosGetTimestampUs(); - - int32_t code = blockDataAssign(pCols, pDataBlock, index); + code = blockDataAssign(pCols, pDataBlock, index); if (code) { return code; } @@ -1620,6 +1633,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) { if (pBlock == NULL){ return; } + int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfOutput; ++i) { SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 05daede777..cd0ca95d3b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1204,8 +1204,12 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { SConfigItem *pItem = NULL; TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "timezone"); - TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + if (0 == strlen(pItem->str)) { + uError("timezone is not set"); + } else { + TAOS_CHECK_RETURN(osSetTimezone(pItem->str)); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); + } TAOS_CHECK_RETURN(cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype, true)); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "locale"); @@ -1216,7 +1220,7 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) { int32_t code = taosSetSystemLocale(locale, charset); if (TSDB_CODE_SUCCESS != code) { - uInfo("failed to set locale %s, since: %s", locale, tstrerror(code)); + uError("failed to set locale:%s, since: %s", locale, tstrerror(code)); char curLocale[TD_LOCALE_LEN] = {0}; char curCharset[TD_CHARSET_LEN] = {0}; taosGetSystemLocale(curLocale, curCharset); @@ -1568,13 +1572,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + SConfig *pCfg = NULL; if (tsCfg == NULL) { - TAOS_CHECK_RETURN(osDefaultInit()); + TAOS_CHECK_GOTO(osDefaultInit(), &lino, _exit); } - SConfig *pCfg = NULL; - TAOS_CHECK_RETURN(cfgInit(&pCfg)); + TAOS_CHECK_GOTO(cfgInit(&pCfg), &lino, _exit); if (tsc) { tsLogEmbedded = 0; @@ -1604,7 +1608,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi SConfigItem *pDebugItem = NULL; TAOS_CHECK_GET_CFG_ITEM(pCfg, pDebugItem, "debugFlag"); - TAOS_CHECK_RETURN(taosSetAllDebugFlag(pCfg, pDebugItem->i32)); + TAOS_CHECK_GOTO(taosSetAllDebugFlag(pCfg, pDebugItem->i32), &lino, _exit); if ((code = taosMulModeMkDir(tsLogDir, 0777, true)) != TSDB_CODE_SUCCESS) { printf("failed to create dir:%s since %s\n", tsLogDir, tstrerror(code)); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 3d758e1fd3..6de9665db9 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -387,13 +387,14 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.supportBatch = 1; rpcInit.batchSize = 8 * 1024; rpcInit.timeToGetConn = tsTimeToGetAvailableConn; + rpcInit.notWaitAvaliableConn = 1; (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { - dError("failed to init dnode rpc client"); - return -1; + dError("failed to init dnode rpc client since:%s", tstrerror(terrno)); + return terrno; } dDebug("dnode rpc client is initialized"); @@ -436,8 +437,8 @@ int32_t dmInitStatusClient(SDnode *pDnode) { pTrans->statusRpc = rpcOpen(&rpcInit); if (pTrans->statusRpc == NULL) { - dError("failed to init dnode rpc status client"); - return TSDB_CODE_OUT_OF_MEMORY; + dError("failed to init dnode rpc status client since %s", tstrerror(terrno)); + return terrno; } dDebug("dnode rpc status client is initialized"); @@ -481,8 +482,8 @@ int32_t dmInitSyncClient(SDnode *pDnode) { pTrans->syncRpc = rpcOpen(&rpcInit); if (pTrans->syncRpc == NULL) { - dError("failed to init dnode rpc sync client"); - return TSDB_CODE_OUT_OF_MEMORY; + dError("failed to init dnode rpc sync client since %s", tstrerror(terrno)); + return terrno; } dDebug("dnode rpc sync client is initialized"); @@ -531,7 +532,7 @@ int32_t dmInitServer(SDnode *pDnode) { (void)taosVersionStrToInt(version, &(rpcInit.compatibilityVer)); pTrans->serverRpc = rpcOpen(&rpcInit); if (pTrans->serverRpc == NULL) { - dError("failed to init dnode rpc server"); + dError("failed to init dnode rpc server since:%s", tstrerror(terrno)); return terrno; } diff --git a/source/dnode/mnode/impl/inc/mndCompact.h b/source/dnode/mnode/impl/inc/mndCompact.h index 43cf78a90e..43996705d2 100644 --- a/source/dnode/mnode/impl/inc/mndCompact.h +++ b/source/dnode/mnode/impl/inc/mndCompact.h @@ -47,6 +47,7 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq); SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId); void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact); +int32_t mndCompactGetDbName(SMnode *pMnode, int32_t compactId, char *dbname, int32_t len); void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index a773c61986..175730b91e 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -46,7 +46,7 @@ const char *mndConsumerStatusName(int status); #define MND_TMQ_NULL_CHECK(c) \ do { \ if (c == NULL) { \ - code = TSDB_CODE_OUT_OF_MEMORY; \ + code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); \ goto END; \ } \ } while (0) diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 8704286826..b72d1386c1 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -30,6 +30,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList); +bool mndDbIsExist(SMnode *pMnode, const char *db); SSdbRaw *mndDbActionEncode(SDbObj *pDb); const char *mndGetDbStr(const char *src); diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index dc6c137455..b56cbef2a1 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -30,6 +30,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic); int32_t mndDropTopicByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); bool mndTopicExistsForDb(SMnode *pMnode, SDbObj *pDb); void mndTopicGetShowName(const char* fullTopic, char* topic); +bool checkTopic(SArray *topics, char *topicName); int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics); diff --git a/source/dnode/mnode/impl/src/mndCompact.c b/source/dnode/mnode/impl/src/mndCompact.c index 8b45b13dd1..aae3a262f4 100644 --- a/source/dnode/mnode/impl/src/mndCompact.c +++ b/source/dnode/mnode/impl/src/mndCompact.c @@ -224,6 +224,21 @@ SCompactObj *mndAcquireCompact(SMnode *pMnode, int64_t compactId) { void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pCompact); + pCompact = NULL; +} + +int32_t mndCompactGetDbName(SMnode *pMnode, int32_t compactId, char *dbname, int32_t len) { + int32_t code = 0; + SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); + if (pCompact == NULL) { + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } + + (void)strncpy(dbname, pCompact->dbname, len); + mndReleaseCompact(pMnode, pCompact); + TAOS_RETURN(code); } // compact db @@ -488,7 +503,7 @@ _OVER: } tFreeSKillCompactReq(&killCompactReq); - sdbRelease(pMnode->pSdb, pCompact); + mndReleaseCompact(pMnode, pCompact); TAOS_RETURN(code); } @@ -640,16 +655,12 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { sdbRelease(pMnode->pSdb, pDetail); } - SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); - if (pCompact == NULL) TAOS_RETURN(code); + char dbname[TSDB_TABLE_FNAME_LEN] = {0}; + TAOS_CHECK_RETURN(mndCompactGetDbName(pMnode, compactId, dbname, TSDB_TABLE_FNAME_LEN)); - SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname); - if (pDb == NULL) { + if (!mndDbIsExist(pMnode, dbname)) { needSave = true; - mWarn("compact:%" PRId32 ", no db exist, set needSave:%s", compactId, pCompact->dbname); - } else { - mndReleaseDb(pMnode, pDb); - pDb = NULL; + mWarn("compact:%" PRId32 ", no db exist, set needSave:%s", compactId, dbname); } if (!needSave) { @@ -666,7 +677,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { } mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); - mndTransSetDbName(pTrans, pCompact->dbname, NULL); + mndTransSetDbName(pTrans, dbname, NULL); pIter = NULL; while (1) { @@ -734,24 +745,20 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { sdbRelease(pMnode->pSdb, pDetail); } - pDb = mndAcquireDb(pMnode, pCompact->dbname); - if (pDb == NULL) { + if (!mndDbIsExist(pMnode, dbname)) { allFinished = true; - mWarn("compact:%" PRId32 ", no db exist, set all finished:%s", compactId, pCompact->dbname); - } else { - mndReleaseDb(pMnode, pDb); - pDb = NULL; + mWarn("compact:%" PRId32 ", no db exist, set all finished:%s", compactId, dbname); } if (allFinished) { - mInfo("compact:%d, all finished", pCompact->compactId); + mInfo("compact:%d, all finished", compactId); pIter = NULL; while (1) { SCompactDetailObj *pDetail = NULL; pIter = sdbFetch(pMnode->pSdb, SDB_COMPACT_DETAIL, pIter, (void **)&pDetail); if (pIter == NULL) break; - if (pDetail->compactId == pCompact->compactId) { + if (pDetail->compactId == compactId) { SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail); if (pCommitRaw == NULL) { mndTransDrop(pTrans); @@ -774,7 +781,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { sdbRelease(pMnode->pSdb, pDetail); } + SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); + if (pCompact == NULL) { + mndTransDrop(pTrans); + code = TSDB_CODE_MND_RETURN_VALUE_NULL; + if (terrno != 0) code = terrno; + TAOS_RETURN(code); + } SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); + mndReleaseCompact(pMnode, pCompact); if (pCommitRaw == NULL) { mndTransDrop(pTrans); code = TSDB_CODE_MND_RETURN_VALUE_NULL; @@ -793,11 +808,9 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) { mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr()); mndTransDrop(pTrans); - sdbRelease(pMnode->pSdb, pCompact); TAOS_RETURN(code); } - sdbRelease(pMnode->pSdb, pCompact); mndTransDrop(pTrans); return 0; } @@ -827,8 +840,8 @@ void mndCompactPullup(SMnode *pMnode) { if ((code = mndSaveCompactProgress(pMnode, pCompact->compactId)) != 0) { mError("compact:%d, failed to save compact progress since %s", pCompact->compactId, tstrerror(code)); } + mndReleaseCompact(pMnode, pCompact); } - mndReleaseCompact(pMnode, pCompact); } taosArrayDestroy(pArray); } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 1a9f808688..6116d2da19 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -87,19 +87,18 @@ END: return code; } -static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser, - bool enableReplay) { +static int32_t validateTopics(STrans* pTrans, SCMSubscribeReq *subscribe, SMnode *pMnode, const char *pUser) { SMqTopicObj *pTopic = NULL; int32_t code = 0; - int32_t numOfTopics = taosArrayGetSize(pTopicList); + int32_t numOfTopics = taosArrayGetSize(subscribe->topicNames); for (int32_t i = 0; i < numOfTopics; i++) { - char *pOneTopic = taosArrayGetP(pTopicList, i); + char *pOneTopic = taosArrayGetP(subscribe->topicNames, i); MND_TMQ_RETURN_CHECK(mndAcquireTopic(pMnode, pOneTopic, &pTopic)); MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic)); MND_TMQ_RETURN_CHECK(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION)); - if (enableReplay) { + if (subscribe->enableReplay) { if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; goto END; @@ -117,6 +116,10 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch mndReleaseDb(pMnode, pDb); } } + char key[TSDB_CONSUMER_ID_LEN] = {0}; + (void)snprintf(key, TSDB_CONSUMER_ID_LEN, "%"PRIx64, subscribe->consumerId); + mndTransSetDbName(pTrans, pTopic->db, key); + MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); mndReleaseTopic(pMnode, pTopic); } return 0; @@ -128,7 +131,6 @@ END: static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { int32_t code = 0; - int32_t lino = 0; SMnode *pMnode = pMsg->info.node; SMqConsumerClearMsg *pClearMsg = pMsg->pCont; SMqConsumerObj *pConsumerNew = NULL; @@ -563,16 +565,19 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; MND_TMQ_RETURN_CHECK(tDeserializeSCMSubscribeReq(msgStr, &subscribe, pMsg->contLen)); - if(taosArrayGetSize(subscribe.topicNames) == 0){ + bool ubSubscribe = (taosArrayGetSize(subscribe.topicNames) == 0); + if(ubSubscribe){ SMqConsumerObj *pConsumerTmp = NULL; MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, subscribe.consumerId, &pConsumerTmp)); mndReleaseConsumer(pMnode, pConsumerTmp); } MND_TMQ_RETURN_CHECK(checkAndSortTopic(pMnode, subscribe.topicNames)); - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, + (ubSubscribe ? TRN_CONFLICT_NOTHING :TRN_CONFLICT_DB_INSIDE), + pMsg, "subscribe"); MND_TMQ_NULL_CHECK(pTrans); - MND_TMQ_RETURN_CHECK(validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay)); + MND_TMQ_RETURN_CHECK(validateTopics(pTrans, &subscribe, pMnode, pMsg->info.conn.user)); MND_TMQ_RETURN_CHECK(buildSubConsumer(pMnode, &subscribe, &pConsumerNew)); MND_TMQ_RETURN_CHECK(mndSetConsumerCommitLogs(pTrans, pConsumerNew)); MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5a7831ac0e..dd3f89c9d0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -398,6 +398,17 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } +bool mndDbIsExist(SMnode *pMnode, const char *db) { + SDbObj *pDb = mndAcquireDb(pMnode, db); + if (pDb == NULL) { + return false; + } else { + mndReleaseDb(pMnode, pDb); + pDb = NULL; + return true; + } +} + static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) { char *pos = strstr(dbName, TS_PATH_DELIMITER); if (pos == NULL) { diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0b7562b923..c6b4de74d1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -533,7 +533,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { return code; } - code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0); + code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code) { taosMemoryFree(buf); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 38fddd8bf0..b4bd9c8847 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -320,7 +320,7 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT return terrno; } - code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); + code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); return terrno; @@ -424,7 +424,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa (void) epsetToStr(&epset, buf, tListLen(buf)); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); - code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); + code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); return code; @@ -484,7 +484,7 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. - code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); + code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); return code; @@ -540,7 +540,8 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. - code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); + code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, + TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != 0) { taosMemoryFree(pReq); return code; @@ -713,7 +714,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa return code; } - code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, 0); + code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pReq); } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 585bf5527d..bff313dbaf 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -650,7 +650,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu goto END; } - mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup); + mndTransSetDbName(pTrans, pOutput->pSub->dbName, pOutput->pSub->key); MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); // 1. redo action: action to all vg @@ -882,7 +882,7 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu rebInput->oldConsumerNum = 0; code = mndCreateSubscription(pMnode, pTopic, key, &rebOutput->pSub); if (code != 0) { - mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, terrstr()); + mError("[rebalance] mq rebalance %s failed create sub since %s, ignore", key, tstrerror(code)); taosRUnLockLatch(&pTopic->lock); mndReleaseTopic(pMnode, pTopic); return code; @@ -1018,37 +1018,37 @@ END: return code; } -//static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { -// void *pIter = NULL; -// SMqConsumerObj *pConsumer = NULL; -// int code = 0; -// while (1) { -// pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); -// if (pIter == NULL) { -// break; -// } -// -// // drop consumer in lost status, other consumers not in lost status already deleted by rebalance -// if (pConsumer->status != MQ_CONSUMER_STATUS_LOST || strcmp(cgroup, pConsumer->cgroup) != 0) { -// sdbRelease(pMnode->pSdb, pConsumer); -// continue; -// } -// int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); -// for (int32_t i = 0; i < sz; i++) { -// char *name = taosArrayGetP(pConsumer->assignedTopics, i); -// if (name && strcmp(topic, name) == 0) { -// MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer)); -// } -// } -// -// sdbRelease(pMnode->pSdb, pConsumer); -// } -// -//END: -// sdbRelease(pMnode->pSdb, pConsumer); -// sdbCancelFetch(pMnode->pSdb, pIter); -// return code; -//} +static int32_t mndCheckConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { + void *pIter = NULL; + SMqConsumerObj *pConsumer = NULL; + int code = 0; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) { + break; + } + + if (strcmp(cgroup, pConsumer->cgroup) != 0) { + sdbRelease(pMnode->pSdb, pConsumer); + continue; + } + + bool found = checkTopic(pConsumer->assignedTopics, topic); + if (found){ + mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", + topic, pConsumer->consumerId, pConsumer->cgroup); + code = TSDB_CODE_MND_CGROUP_USED; + goto END; + } + + sdbRelease(pMnode->pSdb, pConsumer); + } + +END: + sdbRelease(pMnode->pSdb, pConsumer); + sdbCancelFetch(pMnode->pSdb, pIter); + return code; +} static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; @@ -1067,7 +1067,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { return 0; } else { code = TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST; - mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, terrstr()); + mError("topic:%s, cgroup:%s, failed to drop since %s", dropReq.topic, dropReq.cgroup, tstrerror(code)); return code; } } @@ -1075,16 +1075,17 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { taosWLockLatch(&pSub->lock); if (taosHashGetSize(pSub->consumerHash) != 0) { code = TSDB_CODE_MND_CGROUP_USED; - mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); + mError("cgroup:%s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, tstrerror(code)); goto END; } - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pMsg, "drop-cgroup"); MND_TMQ_NULL_CHECK(pTrans); mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); - mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup); + mndTransSetDbName(pTrans, pSub->dbName, NULL); MND_TMQ_RETURN_CHECK(mndTransCheckConflict(pMnode, pTrans)); MND_TMQ_RETURN_CHECK(sendDeleteSubToVnode(pMnode, pSub, pTrans)); + MND_TMQ_RETURN_CHECK(mndCheckConsumerByGroup(pMnode, pTrans, dropReq.cgroup, dropReq.topic)); MND_TMQ_RETURN_CHECK(mndSetDropSubCommitLogs(pMnode, pTrans, pSub)); MND_TMQ_RETURN_CHECK(mndTransPrepare(pMnode, pTrans)); @@ -1371,7 +1372,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons OffsetRows *tmp = taosArrayGet(offsetRows, i); MND_TMQ_NULL_CHECK(tmp); if (tmp->vgId != pVgEp->vgId) { - mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId); + mInfo("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId); continue; } data = tmp; @@ -1395,7 +1396,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t cons pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); MND_TMQ_NULL_CHECK(pColInfo); colDataSetNULL(pColInfo, *numOfRows); - mError("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); + mInfo("mnd show subscriptions: do not find vgId:%d in offsetRows", pVgEp->vgId); } (*numOfRows)++; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 46d5afc145..c27cc04ae8 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -574,7 +574,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { END: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("failed to create topic:%s since %s", createTopicReq.name, terrstr()); + mError("failed to create topic:%s since %s", createTopicReq.name, tstrerror(code)); } mndReleaseTopic(pMnode, pTopic); @@ -602,7 +602,7 @@ END: return code; } -static bool checkTopic(SArray *topics, char *topicName){ +bool checkTopic(SArray *topics, char *topicName){ int32_t sz = taosArrayGetSize(topics); for (int32_t i = 0; i < sz; i++) { char *name = taosArrayGetP(topics, i); @@ -613,44 +613,33 @@ static bool checkTopic(SArray *topics, char *topicName){ return false; } -//static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){ -// int32_t code = 0; -// SSdb *pSdb = pMnode->pSdb; -// void *pIter = NULL; -// SMqConsumerObj *pConsumer = NULL; -// while (1) { -// pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); -// if (pIter == NULL) { -// break; -// } -// -// bool found = checkTopic(pConsumer->assignedTopics, topicName); -// if (found){ -// if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) { -// MND_TMQ_RETURN_CHECK(mndSetConsumerDropLogs(pTrans, pConsumer)); -// sdbRelease(pSdb, pConsumer); -// continue; -// } -// mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", -// topicName, pConsumer->consumerId, pConsumer->cgroup); -// code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; -// goto END; -// } -// -// if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) { -// code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; -// mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", -// topicName, pConsumer->consumerId, pConsumer->cgroup); -// goto END; -// } -// sdbRelease(pSdb, pConsumer); -// } -// -//END: -// sdbRelease(pSdb, pConsumer); -// sdbCancelFetch(pSdb, pIter); -// return code; -//} +static int32_t mndCheckConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){ + int32_t code = 0; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SMqConsumerObj *pConsumer = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); + if (pIter == NULL) { + break; + } + + bool found = checkTopic(pConsumer->assignedTopics, topicName); + if (found){ + mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", + topicName, pConsumer->consumerId, pConsumer->cgroup); + code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; + goto END; + } + + sdbRelease(pSdb, pConsumer); + } + +END: + sdbRelease(pSdb, pConsumer); + sdbCancelFetch(pSdb, pIter); + return code; +} static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic){ // broadcast to all vnode @@ -710,7 +699,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { tFreeSMDropTopicReq(&dropReq); return 0; } else { - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + mError("topic:%s, failed to drop since %s", dropReq.name, tstrerror(code)); tFreeSMDropTopicReq(&dropReq); return code; } @@ -725,7 +714,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { MND_TMQ_RETURN_CHECK(mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic)); MND_TMQ_RETURN_CHECK(mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db)); -// MND_TMQ_RETURN_CHECK(mndDropConsumerByTopic(pMnode, pTrans, dropReq.name)); + MND_TMQ_RETURN_CHECK(mndCheckConsumerByTopic(pMnode, pTrans, dropReq.name)); MND_TMQ_RETURN_CHECK(mndDropSubByTopic(pMnode, pTrans, dropReq.name)); if (pTopic->ntbUid != 0) { @@ -738,7 +727,7 @@ END: mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); if (code != 0) { - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + mError("topic:%s, failed to drop since %s", dropReq.name, tstrerror(code)); tFreeSMDropTopicReq(&dropReq); return code; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 8bb2f11e7c..c4823dc62e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1677,7 +1677,7 @@ int32_t mndAcquireUser(SMnode *pMnode, const char *userName, SUserObj **ppUser) *ppUser = sdbAcquire(pSdb, SDB_USER, userName); if (*ppUser == NULL) { - if (code == TSDB_CODE_SDB_OBJ_NOT_THERE) { + if (terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { code = TSDB_CODE_MND_USER_NOT_EXIST; } else { code = TSDB_CODE_MND_USER_NOT_AVAILABLE; @@ -3149,7 +3149,8 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_ (void)memcpy(rsp.user, pUsers[i].user, TSDB_USER_LEN); (void)taosArrayPush(batchRsp.pArray, &rsp); } - mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + mError("user:%s, failed to auth user since %s", pUsers[i].user, tstrerror(code)); + code = 0; continue; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 050c6d09dc..141aff0337 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -132,6 +132,7 @@ int32_t tqMetaDeleteInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen); int32_t tqMetaCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); int32_t tqMetaDecodeCheckInfo(STqCheckInfo *info, void *pVal, int32_t vLen); int32_t tqMetaDecodeOffsetInfo(STqOffset *info, void *pVal, int32_t vLen); +int32_t tqMetaSaveOffset(STQ* pTq, STqOffset* pOffset); int32_t tqMetaGetHandle(STQ* pTq, const char* key, STqHandle** pHandle); int32_t tqMetaGetOffset(STQ* pTq, const char* subkey, STqOffset** pOffset); int32_t tqMetaTransform(STQ* pTq); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index c45d6e6edf..416438b2ec 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -131,7 +131,7 @@ typedef SVCreateTSmaReq SSmaCfg; SMTbCursor* metaOpenTbCursor(void* pVnode); void metaCloseTbCursor(SMTbCursor* pTbCur); void metaPauseTbCursor(SMTbCursor* pTbCur); -void metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first, int8_t move); +int32_t metaResumeTbCursor(SMTbCursor* pTbCur, int8_t first, int8_t move); int32_t metaTbCursorNext(SMTbCursor* pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor* pTbCur, ETableType jumpTableType); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d4ff980311..1d284aaf1f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -231,6 +231,7 @@ _exit: #if 1 // =================================================== SMTbCursor *metaOpenTbCursor(void *pVnode) { SMTbCursor *pTbCur = NULL; + int32_t code; pTbCur = (SMTbCursor *)taosMemoryCalloc(1, sizeof(*pTbCur)); if (pTbCur == NULL) { @@ -241,7 +242,12 @@ SMTbCursor *metaOpenTbCursor(void *pVnode) { // tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); pTbCur->pMeta = pVnodeObj->pMeta; pTbCur->paused = 1; - metaResumeTbCursor(pTbCur, 1, 0); + code = metaResumeTbCursor(pTbCur, 1, 0); + if (code) { + terrno = code; + taosMemoryFree(pTbCur); + return NULL; + } return pTbCur; } @@ -266,28 +272,39 @@ void metaPauseTbCursor(SMTbCursor *pTbCur) { pTbCur->paused = 1; } } -void metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) { +int32_t metaResumeTbCursor(SMTbCursor *pTbCur, int8_t first, int8_t move) { + int32_t code = 0; + int32_t lino; + if (pTbCur->paused) { metaReaderDoInit(&pTbCur->mr, pTbCur->pMeta, META_READER_LOCK); - (void)tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + code = tdbTbcOpen(((SMeta *)pTbCur->pMeta)->pUidIdx, (TBC **)&pTbCur->pDbc, NULL); + TSDB_CHECK_CODE(code, lino, _exit); if (first) { - (void)tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + code = tdbTbcMoveToFirst((TBC *)pTbCur->pDbc); + TSDB_CHECK_CODE(code, lino, _exit); } else { int c = 1; - (void)tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); + code = tdbTbcMoveTo(pTbCur->pDbc, pTbCur->pKey, pTbCur->kLen, &c); + TSDB_CHECK_CODE(code, lino, _exit); if (c == 0) { if (move) tdbTbcMoveToNext(pTbCur->pDbc); } else if (c < 0) { - (void)tdbTbcMoveToPrev(pTbCur->pDbc); + code = tdbTbcMoveToPrev(pTbCur->pDbc); + TSDB_CHECK_CODE(code, lino, _exit); } else { - (void)tdbTbcMoveToNext(pTbCur->pDbc); + code = tdbTbcMoveToNext(pTbCur->pDbc); + TSDB_CHECK_CODE(code, lino, _exit); } } pTbCur->paused = 0; } + +_exit: + return code; } int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { @@ -355,7 +372,9 @@ _query: version = ((SUidIdxVal *)pData)[0].version; - (void)tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData); + if (tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData) != 0) { + goto _err; + } SMetaEntry me = {0}; tDecoderInit(&dc, pData, nData); @@ -385,7 +404,9 @@ _query: } tDecoderInit(&dc, pData, nData); - (void)tDecodeSSchemaWrapperEx(&dc, &schema); + if (tDecodeSSchemaWrapperEx(&dc, &schema) != 0) { + goto _err; + } pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -588,6 +609,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema **ppTSchema) { int32_t code = 0; + int32_t lino; void *pData = NULL; int nData = 0; @@ -603,7 +625,8 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv skmDbKey.uid = suid ? suid : uid; skmDbKey.sver = INT32_MAX; - (void)tdbTbcOpen(pMeta->pSkmDb, &pSkmDbC, NULL); + code = tdbTbcOpen(pMeta->pSkmDb, &pSkmDbC, NULL); + TSDB_CHECK_CODE(code, lino, _exit); metaRLock(pMeta); if (tdbTbcMoveTo(pSkmDbC, &skmDbKey, sizeof(skmDbKey), &c) < 0) { @@ -1440,6 +1463,11 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal}; info.pTagVal = taosMemoryMalloc(pCur->vLen); + if (!info.pTagVal) { + metaCloseCtbCursor(pCur); + taosHashCleanup(pSepecifiedUidMap); + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(info.pTagVal, pCur->pVal, pCur->vLen); if (taosArrayPush(pUidTagInfo, &info) == NULL) { metaCloseCtbCursor(pCur); @@ -1462,6 +1490,11 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) { STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, *index); if (pTagInfo->pTagVal == NULL) { pTagInfo->pTagVal = taosMemoryMalloc(pCur->vLen); + if (!pTagInfo->pTagVal) { + metaCloseCtbCursor(pCur); + taosHashCleanup(pSepecifiedUidMap); + return TSDB_CODE_OUT_OF_MEMORY; + } memcpy(pTagInfo->pTagVal, pCur->pVal, pCur->vLen); } } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2badeec2c9..596fbf70b0 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -306,12 +306,13 @@ _err: } int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tbUidList) { - void *pKey = NULL; - int nKey = 0; - void *pData = NULL; - int nData = 0; - int c = 0; - int rc = 0; + void *pKey = NULL; + int nKey = 0; + void *pData = NULL; + int nData = 0; + int c = 0; + int rc = 0; + int32_t lino; // check if super table exists rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); @@ -323,7 +324,11 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb // drop all child tables TBC *pCtbIdxc = NULL; - (void)(void)tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + rc = tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + if (rc) { + return (terrno = rc); + } + rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { (void)tdbTbcClose(pCtbIdxc); @@ -379,20 +384,20 @@ _exit: return 0; } -static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { - if (!uids) return; +static int32_t metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { + if (!uids) return TSDB_CODE_INVALID_PARA; int c = 0; void *pKey = NULL; int nKey = 0; TBC *pCtbIdxc = NULL; - (void)tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { (void)tdbTbcClose(pCtbIdxc); metaWLock(pMeta); - return; + return 0; } for (;;) { @@ -405,12 +410,17 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) { break; } - (void)taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)); + if (taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid)) == NULL) { + tdbFree(pKey); + (void)tdbTbcClose(pCtbIdxc); + return terrno; + } } tdbFree(pKey); (void)tdbTbcClose(pCtbIdxc); + return 0; } int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { @@ -425,7 +435,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c = -2; - (void)tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { (void)tdbTbcClose(pUidIdxc); @@ -442,7 +452,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = ((SUidIdxVal *)pData)[0].version; - (void)tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); if (!(ret == 0 && c == 0)) { (void)tdbTbcClose(pUidIdxc); @@ -486,7 +496,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; int8_t col_type = pReq->schemaRow.pSchema[nCols - 1].type; - metaGetSubtables(pMeta, pReq->suid, uids); + TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); (void)tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type); } else if (deltaCol == -1) { int16_t cid = -1; @@ -502,7 +512,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { } if (cid != -1) { - metaGetSubtables(pMeta, pReq->suid, uids); + TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pReq->suid, uids)); (void)tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey); } } @@ -619,7 +629,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { * iterator all pTdDbc by uid and version */ TBC *pCtbIdxc = NULL; - (void)tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { (void)tdbTbcClose(pCtbIdxc); @@ -756,7 +766,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) * iterator all pTdDbc by uid and version */ TBC *pCtbIdxc = NULL; - (void)tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { (void)tdbTbcClose(pCtbIdxc); @@ -1424,7 +1434,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search uid index TBC *pUidIdxc = NULL; - (void)tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); (void)tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -1438,7 +1448,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search table.db TBC *pTbDbc = NULL; - (void)tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); (void)tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -1689,7 +1699,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // search uid index TBC *pUidIdxc = NULL; - (void)tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); (void)tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -1706,7 +1716,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA SDecoder dc2 = {0}; /* get ctbEntry */ - (void)tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); (void)tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -1869,7 +1879,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search uid index TBC *pUidIdxc = NULL; - (void)tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL)); (void)tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -1883,7 +1893,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search table.db TBC *pTbDbc = NULL; - (void)tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL)); (void)tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); if (c != 0) { (void)tdbTbcClose(pUidIdxc); @@ -2018,7 +2028,7 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb * iterator all pTdDbc by uid and version */ TBC *pCtbIdxc = NULL; - (void)tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL)); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { (void)tdbTbcClose(pCtbIdxc); @@ -2157,7 +2167,7 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); TBC *pTagIdxc = NULL; - (void)tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL); + TAOS_CHECK_RETURN(tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL)); int rc = tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); for (;;) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index dce6b19d1f..314a6abdf5 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -463,7 +463,7 @@ int32_t tqProcessVgCommittedInfoReq(STQ* pTq, SRpcMsg* pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - SEncoder encoder; + SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); code = tEncodeMqVgOffset(&encoder, &vgOffset); tEncoderClear(&encoder); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 9e90c1cd59..eb8ff72b55 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -102,6 +102,37 @@ int32_t tqMetaDecodeOffsetInfo(STqOffset* info, void* pVal, int32_t vLen) { return code; } +int32_t tqMetaSaveOffset(STQ* pTq, STqOffset* pOffset) { + void* buf = NULL; + int32_t code = TDB_CODE_SUCCESS; + int32_t vlen; + SEncoder encoder = {0}; + tEncodeSize(tEncodeSTqOffset, pOffset, vlen, code); + if (code < 0) { + goto END; + } + + + buf = taosMemoryCalloc(1, vlen); + if (buf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto END; + } + + tEncoderInit(&encoder, buf, vlen); + code = tEncodeSTqOffset(&encoder, pOffset); + if (code < 0) { + goto END; + } + + TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, pOffset->subKey, strlen(pOffset->subKey), buf, vlen)); + +END: + tEncoderClear(&encoder); + taosMemoryFree(buf); + return code; +} + int32_t tqMetaSaveInfo(STQ* pTq, TTB* ttb, const void* key, int32_t kLen, const void* value, int32_t vLen) { int32_t code = TDB_CODE_SUCCESS; TXN* txn = NULL; @@ -170,7 +201,7 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { int32_t code = TDB_CODE_SUCCESS; int32_t vlen; void* buf = NULL; - SEncoder encoder; + SEncoder encoder = {0}; tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); if (code < 0) { goto END; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 63257d7fab..453327c518 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -44,14 +44,15 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { int64_t ret = 0; int32_t size = 0; + int32_t total = 0; while (1) { if ((ret = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) { if (ret != 0) { code = TSDB_CODE_INVALID_MSG; } - goto END; + break; } - + total += INT_BYTES; size = htonl(size); pMemBuf = taosMemoryCalloc(1, size); if (pMemBuf == NULL) { @@ -64,6 +65,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { goto END; } + total += size; STqOffset offset = {0}; TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size)); code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)); @@ -71,13 +73,22 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) { tDeleteSTqOffset(&offset); goto END; } - TQ_ERR_GO_TO_END(tqMetaSaveInfo(pTq, pTq->pOffsetStore, offset.subKey, strlen(offset.subKey), pMemBuf, size)); - tqInfo("tq: offset restore from file to tdb, subkey:%s", offset.subKey); + tqInfo("tq: offset restore from file to tdb, size:%d, hash size:%d subkey:%s", total, taosHashGetSize(pTq->pOffset), offset.subKey); taosMemoryFree(pMemBuf); pMemBuf = NULL; } + void *pIter = NULL; + while ((pIter = taosHashIterate(pTq->pOffset, pIter))) { + STqOffset* pOffset = (STqOffset*)pIter; + code = tqMetaSaveOffset(pTq, pOffset); + if(code != 0){ + taosHashCancelIterate(pTq->pOffset, pIter); + goto END; + } + } + END: taosCloseFile(&pFile); taosMemoryFree(pMemBuf); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index d072d7199c..4357456790 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -101,6 +101,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* TSDB_CHECK_CODE(code, line, END); qStreamSetSourceExcluded(task, pRequest->sourceExcluded); + uint64_t st = taosGetTimestampMs(); while (1) { SSDataBlock* pDataBlock = NULL; code = getDataBlock(task, pHandle, vgId, &pDataBlock); @@ -160,7 +161,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pRsp->common.blockNum++; totalRows += pDataBlock->info.rows; - if (totalRows >= tmqRowSize) { + if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > 1000)) { break; } } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 51d20be3e0..62105459f8 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -595,7 +595,7 @@ int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, i int32_t len = 0; tEncodeSize(tEncodeSubmitReq, pSubmitReq, len, code); - SEncoder encoder; + SEncoder encoder = {0}; len += sizeof(SSubmitReq2Msg); pBuf = rpcMallocCont(len); @@ -1230,7 +1230,7 @@ int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* return code; } - SEncoder encoder; + SEncoder encoder = {0}; void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, len); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index b8ebb2254b..4b206fc04f 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -349,6 +349,9 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { streamMetaWLock(pStreamMeta); pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL); streamMetaWUnLock(pStreamMeta); + if (pTaskList == NULL) { + return terrno; + } tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 9724973440..b3d2300996 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -347,6 +347,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY); goto END; } + totalMetaRows++; if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > 1000)) { tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8d5fa8b0b3..b7e3bb6a07 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -484,6 +484,7 @@ static void tsdbCachePutBatch(SLastCol *pLastCol, const void *key, size_t klen, code = tsdbCacheSerialize(pLastCol, &rocks_value, &vlen); if (code) { tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + return; } (void)taosThreadMutexLock(&rCache->rMutex); @@ -1143,14 +1144,14 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray code = tsdbCacheSerialize(&lastColTmp, &value, &vlen); if (code) { tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } else { + (void)taosThreadMutexLock(&pTsdb->rCache.rMutex); + + rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen); + + (void)taosThreadMutexUnlock(&pTsdb->rCache.rMutex); } - (void)taosThreadMutexLock(&pTsdb->rCache.rMutex); - - rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen); - - (void)taosThreadMutexUnlock(&pTsdb->rCache.rMutex); - pLastCol = &lastColTmp; SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol)); if (!pTmpLastCol) { @@ -1411,6 +1412,11 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr if (IS_LAST_KEY(idxKey->key)) { if (NULL == lastTmpIndexArray) { lastTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); + if (!lastTmpIndexArray) { + taosArrayDestroy(lastrowTmpIndexArray); + + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } (void)taosArrayPush(lastTmpIndexArray, &(i)); lastColIds[lastIndex] = idxKey->key.cid; @@ -1419,6 +1425,11 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } else { if (NULL == lastrowTmpIndexArray) { lastrowTmpIndexArray = taosArrayInit(num_keys, sizeof(int32_t)); + if (!lastrowTmpIndexArray) { + taosArrayDestroy(lastTmpIndexArray); + + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } } (void)taosArrayPush(lastrowTmpIndexArray, &(i)); lastrowColIds[lastrowIndex] = idxKey->key.cid; @@ -1428,6 +1439,11 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } pTmpColArray = taosArrayInit(lastIndex + lastrowIndex, sizeof(SLastCol)); + if (!pTmpColArray) { + taosArrayDestroy(lastrowTmpIndexArray); + taosArrayDestroy(lastTmpIndexArray); + TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY); + } if (lastTmpIndexArray != NULL) { (void)mergeLastCid(uid, pTsdb, &lastTmpColArray, pr, lastColIds, lastIndex, lastSlotIds); @@ -1510,12 +1526,12 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr code = tsdbCacheSerialize(pLastCol, &value, &vlen); if (code) { tsdbError("tsdb/cache: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + } else { + SLastKey *key = &idxKey->key; + size_t klen = ROCKS_KEY_LEN; + rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); + taosMemoryFree(value); } - - SLastKey *key = &idxKey->key; - size_t klen = ROCKS_KEY_LEN; - rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen); - taosMemoryFree(value); } if (wb) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 38a2dd3ab2..af5b3523e8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -355,7 +355,7 @@ void tsdbCacherowsReaderClose(void* pReader) { return; } - if (p->pSchema != NULL) { + if (p->pSchema != NULL && p->transferBuf != NULL) { for (int32_t i = 0; i < p->pSchema->numOfCols; ++i) { taosMemoryFreeClear(p->transferBuf[i]); } @@ -450,23 +450,27 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 return TSDB_CODE_INVALID_PARA; } + int32_t code = TSDB_CODE_SUCCESS; + bool hasRes = false; + SArray* pRow = NULL; + void** pRes = NULL; SCacheRowsReader* pr = pReader; + int32_t pkBufLen = 0; - int32_t code = TSDB_CODE_SUCCESS; - bool hasRes = false; - SArray* pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); + pr->pReadSnap = NULL; + pRow = taosArrayInit(TARRAY_SIZE(pr->pCidList), sizeof(SLastCol)); if (pRow == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } - void** pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES); + pRes = taosMemoryCalloc(pr->numOfCols, POINTER_BYTES); if (pRes == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } - int32_t pkBufLen = (pr->rowKey.numOfPKs > 0) ? pr->pkColumn.bytes : 0; + pkBufLen = (pr->rowKey.numOfPKs > 0) ? pr->pkColumn.bytes : 0; for (int32_t j = 0; j < pr->numOfCols; ++j) { int32_t bytes = (slotIds[j] == -1) ? 1 : pr->pSchema->columns[slotIds[j]].bytes; @@ -690,6 +694,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 _end: tsdbUntakeReadSnap2((STsdbReader*)pr, pr->pReadSnap, true); + pr->pReadSnap = NULL; + if (pr->pCurFileSet) { pr->pCurFileSet = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index e943ef2442..e55ede560e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -28,7 +28,7 @@ int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t num SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); if (pLoadInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pLoadInfo->blockData[0].sttBlockIndex = -1; @@ -50,9 +50,8 @@ int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t num pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); if (pLoadInfo->aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pLoadInfo); - return code; + return terrno; } pLoadInfo->pSchema = pSchema; @@ -358,7 +357,7 @@ static int32_t tValueDupPayload(SValue *pVal) { char *p = (char *)pVal->pData; char *pBuf = taosMemoryMalloc(pVal->nData); if (pBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy(pBuf, p, pVal->nData); @@ -371,13 +370,15 @@ static int32_t tValueDupPayload(SValue *pVal) { static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo, TStatisBlkArray *pStatisBlkArray, uint64_t suid, const char *id) { int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; void* px = NULL; + int32_t startIndex = 0; + int32_t numOfBlocks = TARRAY2_SIZE(pStatisBlkArray); if (numOfBlocks <= 0) { return code; } - int32_t startIndex = 0; while ((startIndex < numOfBlocks) && (pStatisBlkArray->data[startIndex].maxTbid.suid < suid)) { ++startIndex; } @@ -413,150 +414,113 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl // existed if (i < rows) { - if (pBlockLoadInfo->info.pUid == NULL) { - pBlockLoadInfo->info.pUid = taosArrayInit(rows, sizeof(int64_t)); - pBlockLoadInfo->info.pFirstTs = taosArrayInit(rows, sizeof(int64_t)); - pBlockLoadInfo->info.pLastTs = taosArrayInit(rows, sizeof(int64_t)); - pBlockLoadInfo->info.pCount = taosArrayInit(rows, sizeof(int64_t)); + SSttTableRowsInfo* pInfo = &pBlockLoadInfo->info; - pBlockLoadInfo->info.pFirstKey = taosArrayInit(rows, sizeof(SValue)); - pBlockLoadInfo->info.pLastKey = taosArrayInit(rows, sizeof(SValue)); + if (pInfo->pUid == NULL) { + pInfo->pUid = taosArrayInit(rows, sizeof(int64_t)); + pInfo->pFirstTs = taosArrayInit(rows, sizeof(int64_t)); + pInfo->pLastTs = taosArrayInit(rows, sizeof(int64_t)); + pInfo->pCount = taosArrayInit(rows, sizeof(int64_t)); + + pInfo->pFirstKey = taosArrayInit(rows, sizeof(SValue)); + pInfo->pLastKey = taosArrayInit(rows, sizeof(SValue)); + + if (pInfo->pUid == NULL || pInfo->pFirstTs == NULL || pInfo->pLastTs == NULL || pInfo->pCount == NULL || + pInfo->pFirstKey == NULL || pInfo->pLastKey == NULL) { + code = terrno; + goto _end; + } } if (pStatisBlkArray->data[k].maxTbid.suid == suid) { int32_t size = rows - i; int32_t offset = i * sizeof(int64_t); - px = taosArrayAddBatch(pBlockLoadInfo->info.pUid, tBufferGetDataAt(&block.uids, offset), size); - if (px == NULL) { - return terrno; - } + px = taosArrayAddBatch(pInfo->pUid, tBufferGetDataAt(&block.uids, offset), size); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayAddBatch(pBlockLoadInfo->info.pFirstTs, tBufferGetDataAt(&block.firstKeyTimestamps, offset), size); - if (px == NULL){ - return terrno; - } + px = taosArrayAddBatch(pInfo->pFirstTs, tBufferGetDataAt(&block.firstKeyTimestamps, offset), size); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayAddBatch(pBlockLoadInfo->info.pLastTs, tBufferGetDataAt(&block.lastKeyTimestamps, offset), size); - if (px == NULL){ - return terrno; - } + px = taosArrayAddBatch(pInfo->pLastTs, tBufferGetDataAt(&block.lastKeyTimestamps, offset), size); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayAddBatch(pBlockLoadInfo->info.pCount, tBufferGetDataAt(&block.counts, offset), size); - if (px == NULL){ - return terrno; - } + px = taosArrayAddBatch(pInfo->pCount, tBufferGetDataAt(&block.counts, offset), size); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); if (block.numOfPKs > 0) { SValue vFirst = {0}, vLast = {0}; for (int32_t f = i; f < rows; ++f) { code = tValueColumnGet(&block.firstKeyPKs[0], f, &vFirst); - if (code) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); code = tValueDupPayload(&vFirst); - if (code) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); - px = taosArrayPush(pBlockLoadInfo->info.pFirstKey, &vFirst); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pFirstKey, &vFirst); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); // todo add api to clone the original data code = tValueColumnGet(&block.lastKeyPKs[0], f, &vLast); - if (code) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); code = tValueDupPayload(&vLast); - if (code) { - break; - } + TSDB_CHECK_CODE(code, lino, _end); - px = taosArrayPush(pBlockLoadInfo->info.pLastKey, &vLast); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pLastKey, &vLast); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } } else { SValue vFirst = {0}; for (int32_t j = 0; j < size; ++j) { - px = taosArrayPush(pBlockLoadInfo->info.pFirstKey, &vFirst); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pFirstKey, &vFirst); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayPush(pBlockLoadInfo->info.pLastKey, &vFirst); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pLastKey, &vFirst); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } } } else { STbStatisRecord record = {0}; - while (i < rows) { (void)tStatisBlockGet(&block, i, &record); if (record.suid != suid) { break; } - px = taosArrayPush(pBlockLoadInfo->info.pUid, &record.uid); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pUid, &record.uid); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayPush(pBlockLoadInfo->info.pCount, &record.count); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pCount, &record.count); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayPush(pBlockLoadInfo->info.pFirstTs, &record.firstKey.ts); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pFirstTs, &record.firstKey.ts); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayPush(pBlockLoadInfo->info.pLastTs, &record.lastKey.ts); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pLastTs, &record.lastKey.ts); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); if (record.firstKey.numOfPKs > 0) { SValue s = record.firstKey.pks[0]; code = tValueDupPayload(&s); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - px = taosArrayPush(pBlockLoadInfo->info.pFirstKey, &s); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pFirstKey, &s); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); s = record.lastKey.pks[0]; code = tValueDupPayload(&s); - if (code) { - return code; - } + TSDB_CHECK_CODE(code, lino, _end); - px = taosArrayPush(pBlockLoadInfo->info.pLastKey, &s); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pLastKey, &s); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } else { SValue v = {0}; - px = taosArrayPush(pBlockLoadInfo->info.pFirstKey, &v); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pFirstKey, &v); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); - px = taosArrayPush(pBlockLoadInfo->info.pLastKey, &v); - if (px == NULL) { - return terrno; - } + px = taosArrayPush(pInfo->pLastKey, &v); + TSDB_CHECK_NULL(px, code, lino, _end, terrno); } i += 1; @@ -565,6 +529,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl } } + _end: (void)tStatisBlockDestroy(&block); double el = (taosGetTimestampUs() - st) / 1000.0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 51e3672c56..09ca1cdc84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2434,21 +2434,25 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; if (info.pKeyRangeList == NULL) { + pReader->code = terrno; return false; } int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { + pReader->code = code; return false; } code = initMemDataIterator(pScanInfo, pReader); if (code != TSDB_CODE_SUCCESS) { + pReader->code = code; return false; } code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); if (code != TSDB_CODE_SUCCESS) { + pReader->code = code; return code; } @@ -2461,7 +2465,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan for (int32_t i = 0; i < taosArrayGetSize(info.pKeyRangeList); ++i) { SSttKeyRange* pKeyRange = taosArrayGet(info.pKeyRangeList, i); if (pKeyRange == NULL) { - return TSDB_CODE_INVALID_PARA; + continue; } if (pkCompEx(&pScanInfo->sttRange.skey, &pKeyRange->skey) > 0) { @@ -2766,6 +2770,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SBlockData* pBlockData = &pReader->status.fileBlockData; (void) initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader); + if (pReader->code != 0) { + code = pReader->code; + goto _end; + } while (1) { bool hasBlockData = false; @@ -3180,6 +3188,10 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { } bool hasDataInSttFile = initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + if (pReader->code != TSDB_CODE_SUCCESS) { + return pReader->code; + } + if (!hasDataInSttFile) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { @@ -3273,6 +3285,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + if (pReader->code != 0) { + return pReader->code; + } } TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); @@ -3314,6 +3329,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files (void) initSttBlockReader(pSttBlockReader, pScanInfo, pReader); + if (pReader->code != 0) { + return pReader->code; + } // no data in stt block, no need to proceed. while (hasDataInSttBlock(pScanInfo)) { @@ -3414,6 +3432,10 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t en } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; + if (pBlockScanInfo == NULL || *pBlockScanInfo == NULL) { + return TSDB_CODE_SUCCESS; + } + if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) { bool hasNexTable = moveToNextTable(pUidList, pStatus); @@ -4745,9 +4767,12 @@ void tsdbReaderClose2(STsdbReader* pReader) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; TARRAY2_DESTROY(&pSupInfo->colAggArray, NULL); - for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { - if (pSupInfo->buildBuf[i] != NULL) { - taosMemoryFreeClear(pSupInfo->buildBuf[i]); + + if (pSupInfo->buildBuf) { + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { + if (pSupInfo->buildBuf[i] != NULL) { + taosMemoryFreeClear(pSupInfo->buildBuf[i]); + } } } @@ -4788,6 +4813,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { void* p = pReader->pReadSnap; if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, true); + pReader->pReadSnap = NULL; } (void) tsem_destroy(&pReader->resumeAfterSuspend); @@ -4870,6 +4896,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { void* p = pReader->pReadSnap; if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) { tsdbUntakeReadSnap2(pReader, p, false); + pReader->pReadSnap = NULL; } if (pReader->bFilesetDelimited) { @@ -5762,6 +5789,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs int32_t code = 0; STsdb* pTsdb = pReader->pTsdb; SVersionRange* pRange = &pReader->info.verRange; + *ppSnap = NULL; // lock code = taosThreadMutexLock(&pTsdb->mutex); @@ -5774,7 +5802,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); if (pSnap == NULL) { (void) taosThreadMutexUnlock(&pTsdb->mutex); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -5784,7 +5812,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); if (pSnap->pNode == NULL) { (void) taosThreadMutexUnlock(&pTsdb->mutex); - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; goto _exit; } @@ -5798,8 +5826,10 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs pSnap->pIMem = pTsdb->imem; pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); if (pSnap->pINode == NULL) { + tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true); // unref the previous refed mem + code = terrno; + (void) taosThreadMutexUnlock(&pTsdb->mutex); - code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -5811,28 +5841,33 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs // fs code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray); + if (code) { + if (pSnap->pNode) { + tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true); // unref the previous refed mem + } + + if (pSnap->pINode) { + tsdbUnrefMemTable(pTsdb->imem, pSnap->pINode, true); + } + + (void) taosThreadMutexUnlock(&pTsdb->mutex); + goto _exit; + } // unlock (void) taosThreadMutexUnlock(&pTsdb->mutex); + *ppSnap = pSnap; - if (code == TSDB_CODE_SUCCESS) { - tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); - } + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + return code; _exit: - if (code != TSDB_CODE_SUCCESS) { - tsdbError("vgId:%d take read snapshot failed, code:%s", TD_VID(pTsdb->pVnode), tstrerror(code)); - - *ppSnap = NULL; - if (pSnap) { - if (pSnap->pNode) taosMemoryFree(pSnap->pNode); - if (pSnap->pINode) taosMemoryFree(pSnap->pINode); - taosMemoryFree(pSnap); - } - } else { - *ppSnap = pSnap; + tsdbError("vgId:%d take read snapshot failed, code:%s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); } - return code; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index afadc8d8c2..f70cfff71d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -1141,6 +1141,7 @@ uint64_t ctgGetClusterCacheSize(SCatalog *pCtg); void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardNum, bool *roundDone); void ctgClearAllHandleMeta(int64_t *clearedSize, int64_t *clearedNum, bool *roundDone); void ctgProcessTimerEvent(void *param, void *tmrId); +int32_t ctgBuildUseDbOutput(SUseDbOutput** ppOut, SDBVgInfo* vgInfo); int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 7f57a09460..4caa66445a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -2881,6 +2881,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask* pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { + if (pTask->subTask) { + pMsgCtx->reqType = TDMT_MND_USE_DB; + CTG_ERR_JRET(ctgBuildUseDbOutput((SUseDbOutput**)&pMsgCtx->out, dbCache->vgCache.vgInfo)); + } + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); ctgReleaseVgInfoToCache(pCtg, dbCache); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index c46ded17b8..fa4df70f59 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -2590,6 +2590,22 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) { return TSDB_CODE_SUCCESS; } + +int32_t ctgBuildUseDbOutput(SUseDbOutput** ppOut, SDBVgInfo* vgInfo) { + *ppOut = taosMemoryCalloc(1, sizeof(SUseDbOutput)); + if (NULL == *ppOut) { + CTG_ERR_RET(terrno); + } + + int32_t code = cloneDbVgInfo(vgInfo, &(*ppOut)->dbVgroup); + if (code) { + taosMemoryFreeClear(*ppOut); + CTG_RET(code); + } + + return TSDB_CODE_SUCCESS; +} + uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) { if (!pTsmaInfo) { return 0; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 2811402ea1..11ddc89d4c 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -38,7 +38,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } (*pRsp)->useconds = 0; @@ -289,7 +289,7 @@ static int32_t buildRetension(SArray* pRetension, char **ppRetentions ) { char* p1 = taosMemoryCalloc(1, 100); if(NULL == p1) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t len = 0; @@ -849,7 +849,7 @@ _return: static int32_t buildLocalVariablesResultDataBlock(SSDataBlock** pOutput) { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (NULL == pBlock) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pBlock->info.hasVarCol = true; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 8d9f1fb9cc..3a73c05de2 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -227,7 +227,7 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } int32_t code = 0; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2adc863baf..46fc618f76 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -26,7 +26,7 @@ #define T_LONG_JMP(_obj, _c) \ do { \ - ASSERT((_c) != 1); \ + ASSERT(1); \ longjmp((_obj), (_c)); \ } while (0) @@ -218,6 +218,6 @@ uint64_t calcGroupId(char* pData, int32_t len); SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys); -int32_t extractKeysLen(const SArray* keys); +int32_t extractKeysLen(const SArray* keys, int32_t* pLen); #endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 668d40dd0b..d295e868e9 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -202,6 +202,7 @@ typedef struct SExchangeInfo { SLimitInfo limitInfo; int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo char* pTaskId; + SArray* pFetchRpcHandles; } SExchangeInfo; typedef struct SScanInfo { diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index a6197370e3..093555c9c5 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -147,12 +147,10 @@ _error: if (pInfo != NULL) { destroyAggOperatorInfo(pInfo); } - if (pOperator != NULL) { - cleanupExprSupp(&pOperator->exprSupp); + pOperator->info = NULL; + destroyOperator(pOperator); } - - taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return code; } @@ -180,11 +178,11 @@ static bool nextGroupedResult(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; - if (pOperator->blocking && pAggInfo->hasValidBlock) return false; - - SExprSupp* pSup = &pOperator->exprSupp; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (pOperator->blocking && pAggInfo->hasValidBlock) { + return false; + } + SExprSupp* pSup = &pOperator->exprSupp; int64_t st = taosGetTimestampUs(); int32_t order = pAggInfo->binfo.inputTsOrder; SSDataBlock* pBlock = pAggInfo->pNewGroupBlock; @@ -345,6 +343,7 @@ int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) { static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; + SSDataBlock* pBlock = NULL; if (!tsCountAlwaysReturnValue) { return TSDB_CODE_SUCCESS; } @@ -368,7 +367,6 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc return TSDB_CODE_SUCCESS; } - SSDataBlock* pBlock = NULL; code = createDataBlock(&pBlock); if (code) { return code; @@ -409,12 +407,14 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc for (int32_t i = 0; i < blockDataGetNumOfCols(pBlock); ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, 0); } *ppBlock = pBlock; _end: if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(pBlock); qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); } return code; @@ -453,6 +453,9 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin SResultRow* pResultRow = doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, (char*)&groupId, sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup, true); + if (pResultRow == NULL || pTaskInfo->code != 0) { + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } /* * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 73f770a599..5c8ca49813 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -58,13 +58,16 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchI #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) -static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) { +static int32_t setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode* pScan) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SNode* pNode; int32_t idx = 0; FOREACH(pNode, pScan->pTargets) { if (nodeType(pNode) == QUERY_NODE_COLUMN) { SColumnNode* pCol = (SColumnNode*)pNode; SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); + QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno); pColInfo->info.colId = pCol->colId; } idx++; @@ -72,6 +75,7 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode for (; idx < pBlock->pDataBlock->size; ++idx) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); + QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno); if (pScan->scan.pScanPseudoCols) { FOREACH(pNode, pScan->scan.pScanPseudoCols) { STargetNode* pTarget = (STargetNode*)pNode; @@ -82,6 +86,12 @@ static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SLastRowScanPhysiNode } } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, @@ -98,7 +108,6 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); goto _error; } @@ -126,6 +135,7 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl } else { for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { SColMatchItem* pItem = taosArrayGet(pInfo->matchInfo.pList, i); + QUERY_CHECK_NULL(pItem, code, lino, _error, terrno); if (pItem->isPk) { pInfo->numOfPks += 1; pInfo->pkCol.type = pItem->dataType.type; // only record one primary key @@ -148,10 +158,14 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl for (int i = 0; i < TARRAY_SIZE(pInfo->matchInfo.pList); ++i) { SColMatchItem* pColInfo = taosArrayGet(pInfo->matchInfo.pList, i); + QUERY_CHECK_NULL(pColInfo, code, lino, _error, terrno); + void* tmp = taosArrayPush(pCidList, &pColInfo->colId); QUERY_CHECK_NULL(tmp, code, lino, _error, terrno); if (pInfo->pFuncTypeList != NULL && taosArrayGetSize(pInfo->pFuncTypeList) > i) { - pColInfo->funcType = *(int32_t*)taosArrayGet(pInfo->pFuncTypeList, i); + void* pFuncType = taosArrayGet(pInfo->pFuncTypeList, i); + QUERY_CHECK_NULL(pFuncType, code, lino, _error, terrno); + pColInfo->funcType = *(int32_t*)pFuncType; } } pInfo->pCidList = pCidList; @@ -173,6 +187,7 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL | SCAN_ROW_TYPE(pScanNode->ignoreNull); STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); + if (totalTables) QUERY_CHECK_NULL(pList, code, lino, _error, terrno); uint64_t suid = tableListGetSuid(pTableListInfo); code = pInfo->readHandle.api.cacheFn.openReader(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, @@ -187,13 +202,16 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl code = createOneDataBlock(pInfo->pRes, false, &pInfo->pBufferedRes); QUERY_CHECK_CODE(code, lino, _error); - setColIdForCacheReadBlock(pInfo->pBufferedRes, pScanNode); + code = setColIdForCacheReadBlock(pInfo->pBufferedRes, pScanNode); + QUERY_CHECK_CODE(code, lino, _error); + code = blockDataEnsureCapacity(pInfo->pBufferedRes, capacity); QUERY_CHECK_CODE(code, lino, _error); } else { // by tags pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull); capacity = 1; // only one row output - setColIdForCacheReadBlock(pInfo->pRes, pScanNode); + code = setColIdForCacheReadBlock(pInfo->pRes, pScanNode); + QUERY_CHECK_CODE(code, lino, _error); } initResultSizeInfo(&pOperator->resultInfo, capacity); @@ -227,8 +245,12 @@ _error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } + pInfo->pTableList = NULL; destroyCacheScanOperator(pInfo); - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } @@ -283,10 +305,13 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { if (pInfo->indexOfBufferedRes < pBufRes->info.rows) { for (int32_t i = 0; i < taosArrayGetSize(pBufRes->pDataBlock); ++i) { SColumnInfoData* pCol = taosArrayGet(pRes->pDataBlock, i); + QUERY_CHECK_NULL(pCol, code, lino, _end, terrno); int32_t slotId = pCol->info.slotId; SColumnInfoData* pSrc = taosArrayGet(pBufRes->pDataBlock, slotId); + QUERY_CHECK_NULL(pSrc, code, lino, _end, terrno); SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); + QUERY_CHECK_NULL(pDst, code, lino, _end, terrno); if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) { colDataSetNULL(pDst, 0); @@ -299,7 +324,10 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { } } - pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + void* pUid = taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + QUERY_CHECK_NULL(pUid, code, lino, _end, terrno); + + pRes->info.id.uid = *(tb_uid_t*)pUid; pRes->info.rows = 1; pRes->info.scanFlag = MAIN_SCAN; @@ -368,7 +396,9 @@ int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pInfo->pRes->info.id.groupId = pKeyInfo->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { - pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); + void* pUid = taosArrayGet(pInfo->pUidList, 0); + QUERY_CHECK_NULL(pUid, code, lino, _end, terrno); + pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid; code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, pTaskInfo, NULL); if (code != TSDB_CODE_SUCCESS) { @@ -413,15 +443,15 @@ void destroyCacheScanOperator(void* param) { SCacheRowsScanInfo* pInfo = (SCacheRowsScanInfo*)param; blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pBufferedRes); - taosMemoryFree(pInfo->pSlotIds); - taosMemoryFree(pInfo->pDstSlotIds); + taosMemoryFreeClear(pInfo->pSlotIds); + taosMemoryFreeClear(pInfo->pDstSlotIds); taosArrayDestroy(pInfo->pCidList); taosArrayDestroy(pInfo->pFuncTypeList); taosArrayDestroy(pInfo->pUidList); taosArrayDestroy(pInfo->matchInfo.pList); tableListDestroy(pInfo->pTableList); - if (pInfo->pLastrowReader != NULL) { + if (pInfo->pLastrowReader != NULL && pInfo->readHandle.api.cacheFn.closeReader != NULL) { pInfo->readHandle.api.cacheFn.closeReader(pInfo->pLastrowReader); pInfo->pLastrowReader = NULL; } @@ -436,13 +466,13 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask *pSlotIds = taosMemoryMalloc(numOfCols * sizeof(int32_t)); if (*pSlotIds == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } *pDstSlotIds = taosMemoryMalloc(numOfCols * sizeof(int32_t)); if (*pDstSlotIds == NULL) { - taosMemoryFree(*pSlotIds); - return TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(*pSlotIds); + return terrno; } SSchemaInfo* pSchemaInfo = taosArrayGetLast(pTaskInfo->schemaInfos); @@ -450,6 +480,9 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask for (int32_t i = 0; i < numOfCols; ++i) { SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i); + if (!pColMatch) { + return terrno; + } bool found = false; for (int32_t j = 0; j < pWrapper->nCols; ++j) { /* if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -486,11 +519,16 @@ int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pC for (int32_t i = 0; i < size; ++i) { SColMatchItem* pColInfo = taosArrayGet(pColMatchInfo->pList, i); + if (!pColInfo) { + return terrno; + } int32_t slotId = pColInfo->dstSlotId; SNodeList* pList = pScanNode->scan.node.pOutputDataBlockDesc->pSlots; SSlotDescNode* pDesc = (SSlotDescNode*)nodesListGetNode(pList, slotId); + QUERY_CHECK_NULL(pDesc, code, lino, _end, terrno); + if (pDesc->dataType.type != TSDB_DATA_TYPE_TIMESTAMP) { void* tmp = taosArrayPush(pMatchInfo, pColInfo); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index e382bbc9ed..9019fa0fef 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -65,6 +65,9 @@ static void clearWinStateBuff(SCountWindowResult* pBuff) { pBuff->winRows = 0; } static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) { SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex); + if (!pBuffInfo) { + return NULL; + } int32_t size = taosArrayGetSize(pCountSup->pWinStates); // coverity scan ASSERTS(size > 0, "WinStates is empty"); @@ -79,9 +82,15 @@ static int32_t setCountWindowOutputBuff(SExprSupp* pExprSup, SCountWindowSupp* p int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SCountWindowResult* pBuff = getCountWinStateInfo(pCountSup); + QUERY_CHECK_NULL(pBuff, code, lino, _end, terrno); (*pResult) = &pBuff->row; code = setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); (*ppResBuff) = pBuff; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -99,7 +108,8 @@ void doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SCountWindowOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->binfo.pRes; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); - TSKEY* tsCols = (TSKEY*)pColInfoData->pData; + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + TSKEY* tsCols = (TSKEY*)pColInfoData->pData; for (int32_t i = 0; i < pBlock->info.rows;) { SCountWindowResult* pBuffInfo = NULL; @@ -140,6 +150,13 @@ void doCountWindowAggImpl(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } i += step; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } } static void buildCountResult(SExprSupp* pExprSup, SCountWindowSupp* pCountSup, SExecTaskInfo* pTaskInfo, @@ -277,10 +294,11 @@ int32_t createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy SSDataBlock* pResBlock = createDataBlockFromDescNode(pCountWindowNode->window.node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); + initBasicInfo(&pInfo->binfo, pResBlock); + code = blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); - initBasicInfo(&pInfo->binfo, pResBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->binfo.inputTsOrder = physiNode->inputTsOrder; pInfo->binfo.outputTsOrder = physiNode->outputTsOrder; @@ -324,7 +342,10 @@ _error: destroyCountWindowOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index fcd202e221..d4e5dedd20 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -110,11 +110,11 @@ int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy SSDataBlock* pResBlock = createDataBlockFromDescNode(pEventWindowNode->window.node.pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); + initBasicInfo(&pInfo->binfo, pResBlock); code = blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _error); - initBasicInfo(&pInfo->binfo, pResBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->binfo.inputTsOrder = physiNode->inputTsOrder; pInfo->binfo.outputTsOrder = physiNode->outputTsOrder; @@ -145,7 +145,10 @@ _error: destroyEWindowOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -284,6 +287,7 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p SSDataBlock* pRes = pInfo->binfo.pRes; int64_t gid = pBlock->info.id.groupId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + QUERY_CHECK_NULL(pColInfoData, code, lino, _return, terrno); TSKEY* tsList = (TSKEY*)pColInfoData->pData; SWindowRowsSup* pRowSup = &pInfo->winSup; SColumnInfoData *ps = NULL, *pe = NULL; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 2f018d7a69..e4ebb74252 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -52,7 +52,7 @@ static void setAllSourcesCompleted(SOperatorInfo* pOperator); static int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code); static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo, int32_t sourceIndex); -static int32_t getCompletedSources(const SArray* pArray); +static int32_t getCompletedSources(const SArray* pArray, int32_t* pRes); static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); @@ -65,8 +65,14 @@ static int32_t exchangeWait(SOperatorInfo* pOperator, SExchangeInfo* pExchangeIn static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { int32_t code = 0; + int32_t lino = 0; size_t totalSources = taosArrayGetSize(pExchangeInfo->pSourceDataInfo); - int32_t completed = getCompletedSources(pExchangeInfo->pSourceDataInfo); + int32_t completed = 0; + code = getCompletedSources(pExchangeInfo->pSourceDataInfo, &completed); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } if (completed == totalSources) { setAllSourcesCompleted(pOperator); return; @@ -84,6 +90,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn for (int32_t i = 0; i < totalSources; ++i) { pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + QUERY_CHECK_NULL(pDataInfo, code, lino, _error, terrno); if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { continue; } @@ -100,6 +107,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn tmemory_barrier(); SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index); + QUERY_CHECK_NULL(pSource, code, lino, _error, terrno); // todo SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; @@ -159,7 +167,12 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn return; } // end loop - int32_t complete1 = getCompletedSources(pExchangeInfo->pSourceDataInfo); + int32_t complete1 = 0; + code = getCompletedSources(pExchangeInfo->pSourceDataInfo, &complete1); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } if (complete1 == totalSources) { qDebug("all sources are completed, %s", GET_TASKID(pTaskInfo)); return; @@ -285,13 +298,13 @@ _end: pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } - (*ppRes) = NULL; + (*ppRes) = NULL; return code; } static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SSDataBlock* pRes = NULL; - int32_t code = loadRemoteDataNext(pOperator, &pRes); + int32_t code = loadRemoteDataNext(pOperator, &pRes); return pRes; } @@ -333,6 +346,14 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* qError("%s invalid number: %d of sources in exchange operator", id, (int32_t)numOfSources); return TSDB_CODE_INVALID_PARA; } + pInfo->pFetchRpcHandles = taosArrayInit(numOfSources, sizeof(int64_t)); + if (!pInfo->pFetchRpcHandles) { + return terrno; + } + void* ret = taosArrayReserve(pInfo->pFetchRpcHandles, numOfSources); + if (!ret) { + return terrno; + } pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode)); if (pInfo->pSources == NULL) { @@ -350,9 +371,13 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* for (int32_t i = 0; i < numOfSources; ++i) { SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)nodesListGetNode((SNodeList*)pExNode->pSrcEndPoints, i); - void* tmp = taosArrayPush(pInfo->pSources, pNode); + if (!pNode) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return TSDB_CODE_OUT_OF_MEMORY; + } + void* tmp = taosArrayPush(pInfo->pSources, pNode); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return TSDB_CODE_OUT_OF_MEMORY; } SExchangeSrcIndex idx = {.srcIdx = i, .inUseIdx = -1}; @@ -367,6 +392,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); + return initDataSource(numOfSources, pInfo, id); } @@ -374,7 +400,7 @@ int32_t createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNo SOperatorInfo** pOptrInfo) { QRY_OPTR_CHECK(pOptrInfo); - int32_t code = 0; + int32_t code = 0; int32_t lino = 0; SExchangeInfo* pInfo = taosMemoryCalloc(1, sizeof(SExchangeInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -391,6 +417,8 @@ int32_t createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNo QUERY_CHECK_CODE(code, lino, _error); pInfo->pDummyBlock = createDataBlockFromDescNode(pExNode->node.pOutputDataBlockDesc); + QUERY_CHECK_NULL(pInfo->pDummyBlock, code, lino, _error, terrno); + pInfo->pResultBlockList = taosArrayInit(64, POINTER_BYTES); QUERY_CHECK_NULL(pInfo->pResultBlockList, code, lino, _error, terrno); pInfo->pRecycledBlocks = taosArrayInit(64, POINTER_BYTES); @@ -424,7 +452,10 @@ _error: doDestroyExchangeOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -449,6 +480,14 @@ void freeSourceDataInfo(void* p) { void doDestroyExchangeOperatorInfo(void* param) { SExchangeInfo* pExInfo = (SExchangeInfo*)param; + for (int32_t i = 0; i < pExInfo->pFetchRpcHandles->size; ++i) { + int64_t* pRpcHandle = taosArrayGet(pExInfo->pFetchRpcHandles, i); + if (*pRpcHandle > 0) { + SDownstreamSourceNode* pSource = taosArrayGet(pExInfo->pSources, i); + (void)asyncFreeConnById(pExInfo->pTransporter, *pRpcHandle); + } + } + taosArrayDestroy(pExInfo->pFetchRpcHandles); taosArrayDestroy(pExInfo->pSources); taosArrayDestroyEx(pExInfo->pSourceDataInfo, freeSourceDataInfo); @@ -476,7 +515,12 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } int32_t index = pWrapper->sourceIndex; + int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, index); + *pRpcHandle = -1; SSourceDataInfo* pSourceDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, index); + if (!pSourceDataInfo) { + return terrno; + } if (code == TSDB_CODE_SUCCESS) { pSourceDataInfo->pRsp = pMsg->pData; @@ -547,12 +591,20 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); + if (!pDataInfo) { + return terrno; + } + if (EX_SOURCE_DATA_NOT_READY != pDataInfo->status) { return TSDB_CODE_SUCCESS; } pDataInfo->status = EX_SOURCE_DATA_STARTED; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index); + if (!pSource) { + return terrno; + } + pDataInfo->startTime = taosGetTimestampUs(); size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); @@ -638,6 +690,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int64_t transporterId = 0; code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo); QUERY_CHECK_CODE(code, lino, _end); + int64_t* pRpcHandle = taosArrayGet(pExchangeInfo->pFetchRpcHandles, sourceIndex); + *pRpcHandle = transporterId; } _end: @@ -656,11 +710,12 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t } int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + SSDataBlock* pBlock = NULL; if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); - code = blockDecode(pRes, pData, (const char**) pNextStart); + code = blockDecode(pRes, pData, (const char**)pNextStart); if (code) { return code; } @@ -680,7 +735,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo pStart += sizeof(SSysTableSchema); } - SSDataBlock* pBlock = NULL; + pBlock = NULL; code = createDataBlock(&pBlock); QUERY_CHECK_CODE(code, lino, _end); @@ -705,10 +760,12 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo QUERY_CHECK_CODE(code, lino, _end); blockDataDestroy(pBlock); + pBlock = NULL; } _end: if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(pBlock); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; @@ -727,18 +784,26 @@ void setAllSourcesCompleted(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); } -int32_t getCompletedSources(const SArray* pArray) { - size_t total = taosArrayGetSize(pArray); +int32_t getCompletedSources(const SArray* pArray, int32_t* pRes) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; + size_t total = taosArrayGetSize(pArray); int32_t completed = 0; for (int32_t k = 0; k < total; ++k) { SSourceDataInfo* p = taosArrayGet(pArray, k); + QUERY_CHECK_NULL(p, code, lino, _end, terrno); if (p->status == EX_SOURCE_DATA_EXHAUSTED) { completed += 1; } } - return completed; + *pRes = completed; +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { @@ -855,6 +920,11 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { } SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + if (!pDataInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } pDataInfo->status = EX_SOURCE_DATA_NOT_READY; code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); @@ -870,6 +940,11 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { } SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + if (!pSource) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } if (pDataInfo->code != TSDB_CODE_SUCCESS) { qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), @@ -941,6 +1016,10 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic dataInfo.taskId = pExchangeInfo->pTaskId; dataInfo.index = pIdx->srcIdx; dataInfo.pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL); + if (dataInfo.pSrcUidList == NULL) { + return terrno; + } + dataInfo.srcOpType = pBasicParam->srcOpType; dataInfo.tableSeq = pBasicParam->tableSeq; @@ -952,10 +1031,17 @@ int32_t addSingleExchangeSource(SOperatorInfo* pOperator, SExchangeOperatorBasic pIdx->inUseIdx = taosArrayGetSize(pExchangeInfo->pSourceDataInfo) - 1; } else { SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pIdx->inUseIdx); + if (!pDataInfo) { + return terrno; + } if (pDataInfo->status == EX_SOURCE_DATA_EXHAUSTED) { pDataInfo->status = EX_SOURCE_DATA_NOT_READY; } pDataInfo->pSrcUidList = taosArrayDup(pBasicParam->uidList, NULL); + if (pDataInfo->pSrcUidList == NULL) { + return terrno; + } + pDataInfo->srcOpType = pBasicParam->srcOpType; pDataInfo->tableSeq = pBasicParam->tableSeq; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 0ca9908fb1..127e0f18f1 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -233,6 +233,13 @@ SArray* createSortInfo(SNodeList* pNodeList) { for (int32_t i = 0; i < numOfCols; ++i) { SOrderByExprNode* pSortKey = (SOrderByExprNode*)nodesListGetNode(pNodeList, i); + if (!pSortKey) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + taosArrayDestroy(pList); + pList = NULL; + terrno = TSDB_CODE_OUT_OF_MEMORY; + break; + } SBlockOrderInfo bi = {0}; bi.order = (pSortKey->order == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; bi.nullFirst = (pSortKey->nullOrder == NULL_ORDER_FIRST); @@ -267,6 +274,13 @@ SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); + if (!pDescNode) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + blockDataDestroy(pBlock); + pBlock = NULL; + terrno = TSDB_CODE_INVALID_PARA; + break; + } SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; @@ -290,9 +304,17 @@ int32_t prepareDataBlockBuf(SSDataBlock* pDataBlock, SColMatchInfo* pMatchInfo) for (int32_t i = 0; i < taosArrayGetSize(pMatchInfo->pList); ++i) { SColMatchItem* pItem = taosArrayGet(pMatchInfo->pList, i); + if (!pItem) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (pItem->isPk) { SColumnInfoData* pInfoData = taosArrayGet(pDataBlock->pDataBlock, pItem->dstSlotId); + if (!pInfoData) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } pBlockInfo->pks[0].type = pInfoData->info.type; pBlockInfo->pks[1].type = pInfoData->info.type; @@ -652,6 +674,7 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); + QUERY_CHECK_NULL(pkeyInfo, code, lino, end, terrno); STUidTagInfo info = {.uid = pkeyInfo->uid}; void* tmp = taosArrayPush(pUidTagList, &info); QUERY_CHECK_NULL(tmp, code, lino, end, terrno); @@ -706,6 +729,7 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf if (nodeType(pNode) == QUERY_NODE_COLUMN) { SColumnNode* pSColumnNode = (SColumnNode*)pNode; SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, pSColumnNode->slotId); + QUERY_CHECK_NULL(pColInfo, code, lino, end, terrno); code = colDataAssign(output.columnData, pColInfo, rows, NULL); } else if (nodeType(pNode) == QUERY_NODE_VALUE) { continue; @@ -749,6 +773,7 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf for (int i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + QUERY_CHECK_NULL(info, code, lino, end, terrno); char* isNull = (char*)keyBuf; char* pStart = (char*)keyBuf + sizeof(int8_t) * LIST_LENGTH(group); @@ -801,6 +826,8 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); + QUERY_CHECK_NULL(tableList, code, lino, end, terrno); + code = pAPI->metaFn.metaPutTbGroupToCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), tableList, taosArrayGetSize(tableList) * sizeof(STableKeyInfo)); @@ -867,12 +894,15 @@ static SArray* getTableNameList(const SNodeListNode* pList) { // remove the duplicates SArray* pNewList = taosArrayInit(taosArrayGetSize(pTbList), sizeof(void*)); QUERY_CHECK_NULL(pNewList, code, lino, _end, terrno); - void* tmp = taosArrayPush(pNewList, taosArrayGet(pTbList, 0)); + void* tmpTbl = taosArrayGet(pTbList, 0); + QUERY_CHECK_NULL(tmpTbl, code, lino, _end, terrno); + void* tmp = taosArrayPush(pNewList, tmpTbl); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); for (int32_t i = 1; i < numOfTables; ++i) { char** name = taosArrayGetLast(pNewList); char** nameInOldList = taosArrayGet(pTbList, i); + QUERY_CHECK_NULL(nameInOldList, code, lino, _end, terrno); if (strcmp(*name, *nameInOldList) == 0) { continue; } @@ -999,6 +1029,10 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S for (int i = 0; i < numOfExisted; i++) { STUidTagInfo* pTInfo = taosArrayGet(pExistedUidList, i); + if (!pTInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } int32_t tempRes = taosHashPut(uHash, &pTInfo->uid, sizeof(uint64_t), &i, sizeof(i)); if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(tempRes)); @@ -1050,7 +1084,9 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S for (int32_t i = 0; i < taosArrayGetSize(pColList); ++i) { SColumnInfoData colInfo = {0}; - colInfo.info = *(SColumnInfo*)taosArrayGet(pColList, i); + void* tmp = taosArrayGet(pColList, i); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); + colInfo.info = *(SColumnInfo*)tmp; code = blockDataAppendColInfo(pResBlock, &colInfo); QUERY_CHECK_CODE(code, lino, _end); } @@ -1058,7 +1094,7 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S code = blockDataEnsureCapacity(pResBlock, numOfTables); if (code != TSDB_CODE_SUCCESS) { terrno = code; - taosMemoryFree(pResBlock); + blockDataDestroy(pResBlock); return NULL; } @@ -1068,9 +1104,11 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S for (int32_t i = 0; i < numOfTables; i++) { STUidTagInfo* p1 = taosArrayGet(pUidTagList, i); + QUERY_CHECK_NULL(p1, code, lino, _end, terrno); for (int32_t j = 0; j < numOfCols; j++) { SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); + QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno); if (pColInfo->info.colId == -1) { // tbname char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -1128,7 +1166,7 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S _end: if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pResBlock); + blockDataDestroy(pResBlock); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); terrno = code; return NULL; @@ -1144,7 +1182,12 @@ static int32_t doSetQualifiedUid(STableListInfo* pListInfo, SArray* pUidList, co int32_t numOfTables = taosArrayGetSize(pUidTagList); for (int32_t i = 0; i < numOfTables; ++i) { if (pResultList[i]) { - uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid; + STUidTagInfo* tmpTag = (STUidTagInfo*)taosArrayGet(pUidTagList, i); + if (!tmpTag) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } + uint64_t uid = tmpTag->uid; qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResultList[i]); info.uid = uid; @@ -1174,6 +1217,10 @@ static int32_t copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { for (int32_t i = 0; i < numOfExisted; ++i) { uint64_t* uid = taosArrayGet(pUidList, i); + if (!uid) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } STUidTagInfo info = {.uid = *uid}; void* tmp = taosArrayPush(pUidTagList, &info); if (!tmp) { @@ -1242,6 +1289,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN for (int32_t i = 0; i < numOfRows; ++i) { STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i); + QUERY_CHECK_NULL(pInfo, code, lino, end, terrno); void* tmp = taosArrayPush(pUidList, &pInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, end, terrno); } @@ -1392,7 +1440,9 @@ int32_t getTableList(void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, S *(int32_t*)pPayload = numOfTables; if (numOfTables > 0) { - memcpy(pPayload + sizeof(int32_t), taosArrayGet(pUidList, 0), numOfTables * sizeof(uint64_t)); + void* tmp = taosArrayGet(pUidList, 0); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); + memcpy(pPayload + sizeof(int32_t), tmp, numOfTables * sizeof(uint64_t)); } code = pStorageAPI->metaFn.putCachedTableList(pVnode, pScanNode->suid, context.digest, tListLen(context.digest), @@ -1408,7 +1458,9 @@ _end: if (!listAdded) { numOfTables = taosArrayGetSize(pUidList); for (int i = 0; i < numOfTables; i++) { - STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(pUidList, i), .groupId = 0}; + void* tmp = taosArrayGet(pUidList, i); + QUERY_CHECK_NULL(tmp, code, lino, _error, terrno); + STableKeyInfo info = {.uid = *(uint64_t*)tmp, .groupId = 0}; void* p = taosArrayPush(pListInfo->pTableList, &info); if (p == NULL) { @@ -1429,19 +1481,29 @@ _error: } int32_t qGetTableList(int64_t suid, void* pVnode, void* node, SArray** tableList, void* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SSubplan* pSubplan = (SSubplan*)node; SScanPhysiNode pNode = {0}; pNode.suid = suid; pNode.uid = suid; pNode.tableType = TSDB_SUPER_TABLE; + STableListInfo* pTableListInfo = tableListCreate(); + QUERY_CHECK_NULL(pTableListInfo, code, lino, _end, terrno); uint8_t digest[17] = {0}; - int code = + code = getTableList(pVnode, &pNode, pSubplan ? pSubplan->pTagCond : NULL, pSubplan ? pSubplan->pTagIndexCond : NULL, pTableListInfo, digest, "qGetTableList", &((SExecTaskInfo*)pTaskInfo)->storageAPI); + QUERY_CHECK_CODE(code, lino, _end); *tableList = pTableListInfo->pTableList; pTableListInfo->pTableList = NULL; tableListDestroy(pTableListInfo); + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } return code; } @@ -1549,6 +1611,11 @@ SArray* makeColumnArrayFromList(SNodeList* pNodeList) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnNode* pColNode = (SColumnNode*)nodesListGetNode(pNodeList, i); + if (!pColNode) { + taosArrayDestroy(pList); + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return NULL; + } // todo extract method SColumn c = {0}; @@ -1586,6 +1653,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); + QUERY_CHECK_NULL(pNode, code, lino, _end, terrno); if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; @@ -1605,6 +1673,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod int32_t num = LIST_LENGTH(pOutputNodeList->pSlots); for (int32_t i = 0; i < num; ++i) { SSlotDescNode* pNode = (SSlotDescNode*)nodesListGetNode(pOutputNodeList->pSlots, i); + QUERY_CHECK_NULL(pNode, code, lino, _end, terrno); // todo: add reserve flag check // it is a column reserved for the arithmetic expression calculation @@ -1616,6 +1685,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchItem* info = NULL; for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { info = taosArrayGet(pList, j); + QUERY_CHECK_NULL(info, code, lino, _end, terrno); if (info->dstSlotId == pNode->slotId) { break; } @@ -1692,8 +1762,11 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { SDataType* pType = &pColNode->node.resType; pExp->base.resSchema = createResSchema(pType->type, pType->bytes, slotId, pType->scale, pType->precision, pColNode->colName); + pExp->base.pParam[0].pCol = createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); + QUERY_CHECK_NULL(pExp->base.pParam[0].pCol, code, lino, _end, terrno); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; } else if (type == QUERY_NODE_VALUE) { pExp->pExpr->nodeType = QUERY_NODE_VALUE; @@ -1755,12 +1828,14 @@ int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { for (int32_t j = 0; j < numOfParam; ++j) { SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + QUERY_CHECK_NULL(p1, code, lino, _end, terrno); if (p1->type == QUERY_NODE_COLUMN) { SColumnNode* pcn = (SColumnNode*)p1; pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; pExp->base.pParam[j].pCol = createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); + QUERY_CHECK_NULL(pExp->base.pParam[j].pCol, code, lino, _end, terrno); } else if (p1->type == QUERY_NODE_VALUE) { SValueNode* pvn = (SValueNode*)p1; pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; @@ -1830,6 +1905,7 @@ SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs) { int32_t code = createExprFromOneNode(pExp, nodesListGetNode(pNodeList, i), i + UD_TAG_COLUMN_INDEX); if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pExprs); + terrno = code; qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); return NULL; } @@ -1865,6 +1941,10 @@ int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo** } else { pTargetNode = (STargetNode*)nodesListGetNode(pGroupKeys, i - numOfFuncs); } + if (!pTargetNode) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } SExprInfo* pExp = &pExprs[i]; code = createExprFromTargetNode(pExp, pTargetNode); @@ -2042,10 +2122,20 @@ int32_t relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SAr int32_t i = 0, j = 0; while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) { SColumnInfoData* p = taosArrayGet(pCols, i); - SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, j); + if (!p) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } + SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, j); + if (!pmInfo) { + return terrno; + } if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId); + if (!pDst) { + return terrno; + } code = colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); @@ -2119,6 +2209,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi int32_t j = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pTableScanNode->scan.pScanCols, i); + if (!pNode) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; if (pColNode->colType == COLUMN_TYPE_TAG) { continue; @@ -2309,6 +2403,10 @@ int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t st for (int32_t i = startIndex; i < numOfTables; ++i) { STableKeyInfo* p = taosArrayGet(pTableList->pTableList, i); + if (!p) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return -1; + } if (p->uid == uid) { return i; } @@ -2324,7 +2422,9 @@ void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psu uint64_t tableListGetTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); - ASSERT(pTableList->map != NULL && slot != NULL); + if (slot == NULL) { + return -1; + } STableKeyInfo* pKeyInfo = taosArrayGet(pTableList->pTableList, *slot); ASSERT(pKeyInfo->uid == tableUid); @@ -2407,11 +2507,10 @@ bool oneTableForEachGroup(const STableListInfo* pTableList) { return pTableList- STableListInfo* tableListCreate() { STableListInfo* pListInfo = taosMemoryCalloc(1, sizeof(STableListInfo)); if (pListInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pListInfo->remainGroups = NULL; + pListInfo->remainGroups = NULL; pListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); if (pListInfo->pTableList == NULL) { goto _error; @@ -2427,7 +2526,6 @@ STableListInfo* tableListCreate() { _error: tableListDestroy(pListInfo); - terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -2437,7 +2535,6 @@ void tableListDestroy(STableListInfo* pTableListInfo) { } taosArrayDestroy(pTableListInfo->pTableList); - pTableListInfo->pTableList = NULL; taosMemoryFreeClear(pTableListInfo->groupOffset); taosHashCleanup(pTableListInfo->map); @@ -2481,22 +2578,30 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo) { } STableKeyInfo* pInfo = taosArrayGet(pTableListInfo->pTableList, 0); + if (!pInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } uint64_t gid = pInfo->groupId; int32_t start = 0; void* tmp = taosArrayPush(pList, &start); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; } for (int32_t i = 1; i < size; ++i) { pInfo = taosArrayGet(pTableListInfo->pTableList, i); + if (!pInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (pInfo->groupId != gid) { tmp = taosArrayPush(pList, &i); if (!tmp) { - qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); - return TSDB_CODE_OUT_OF_MEMORY; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; } gid = pInfo->groupId; } @@ -2534,6 +2639,10 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* for (int i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + if (!info) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } info->groupId = groupByTbname ? info->uid : 0; int32_t tempRes = taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), @@ -2546,6 +2655,10 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* } else { for (int32_t i = 0; i < numOfTables; i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + if (!info) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } info->groupId = groupByTbname ? info->uid : 0; } } @@ -2589,6 +2702,10 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* size_t size = taosArrayGetSize(pTableListInfo->pTableList); for (int32_t i = 0; i < size; ++i) { STableKeyInfo* p = taosArrayGet(pTableListInfo->pTableList, i); + if (!p) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } int32_t tempRes = taosHashPut(pTableListInfo->map, &p->uid, sizeof(uint64_t), &i, sizeof(int32_t)); if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(tempRes)); @@ -2814,13 +2931,22 @@ SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys) { return ret; } -int32_t extractKeysLen(const SArray* keys) { +int32_t extractKeysLen(const SArray* keys, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; int32_t len = 0; int32_t keyNum = taosArrayGetSize(keys); for (int32_t i = 0; i < keyNum; ++i) { SColumn* pCol = (SColumn*)taosArrayGet(keys, i); + QUERY_CHECK_NULL(pCol, code, lino, _end, terrno); len += pCol->bytes; } len += sizeof(int8_t) * keyNum; // null flag - return len; + *pLen = len; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ef76f14aa9..dc0baebd66 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -392,6 +392,7 @@ static int32_t filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S pAPI->metaReaderFn.initReader(&mr, pScanInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); for (int32_t i = 0; i < numOfUids; ++i) { uint64_t* id = (uint64_t*)taosArrayGet(tableIdList, i); + QUERY_CHECK_NULL(id, code, lino, _end, terrno); int32_t code = pAPI->metaReaderFn.getTableEntryByUid(&mr, *id); if (code != TSDB_CODE_SUCCESS) { @@ -493,6 +494,12 @@ int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableI for (int32_t i = 0; i < numOfQualifiedTables; ++i) { uint64_t* uid = taosArrayGet(qa, i); + if (!uid) { + taosMemoryFree(keyBuf); + taosArrayDestroy(qa); + taosWUnLockLatch(&pTaskInfo->lock); + return terrno; + } STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; if (bufLen > 0) { @@ -547,6 +554,10 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table } SSchemaInfo* pSchemaInfo = taosArrayGet(pTaskInfo->schemaInfos, idx); + if (!pSchemaInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } *sversion = pSchemaInfo->sw->version; *tversion = pSchemaInfo->tversion; @@ -704,7 +715,9 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY); p = p1; } else { - p = *(SSDataBlock**)taosArrayGet(pTaskInfo->pResultBlockList, blockIndex); + void* tmp = taosArrayGet(pTaskInfo->pResultBlockList, blockIndex); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); + p = *(SSDataBlock**)tmp; code = copyDataBlock(p, pRes); QUERY_CHECK_CODE(code, lino, _end); } @@ -752,7 +765,9 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo) { size_t num = taosArrayGetSize(pList); for (int32_t i = 0; i < num; ++i) { SSDataBlock** p = taosArrayGet(pTaskInfo->pResultBlockList, i); - blockDataDestroy(*p); + if (p) { + blockDataDestroy(*p); + } } taosArrayClear(pTaskInfo->pResultBlockList); @@ -866,6 +881,10 @@ void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { int32_t num = taosArrayGetSize(pTaskInfo->stopInfo.pStopInfo); for (int32_t i = 0; i < num; ++i) { SExchangeOpStopInfo* pStop = taosArrayGet(pTaskInfo->stopInfo.pStopInfo, i); + if (!pStop) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + continue; + } SExchangeInfo* pExchangeInfo = taosAcquireRef(exchangeObjRefPool, pStop->refId); if (pExchangeInfo) { (void)tsem_post(&pExchangeInfo->ready); @@ -1274,6 +1293,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (uid == 0) { if (numOfTables != 0) { STableKeyInfo* tmp = tableListGetInfo(pTableListInfo, 0); + if (!tmp) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + taosRUnLockLatch(&pTaskInfo->lock); + return terrno; + } if (tmp) uid = tmp->uid; ts = INT64_MIN; pScanInfo->currentTable = 0; @@ -1403,6 +1427,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } STableKeyInfo* pList = tableListGetInfo(pTableListInfo, 0); + if (!pList) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + tDeleteSchemaWrapper(mtInfo.schema); + return code; + } int32_t size = tableListGetSize(pTableListInfo); code = pTaskInfo->storageAPI.tsdReader.tsdReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, @@ -1487,6 +1516,7 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { int32_t numOfTables = tableListGetSize(pTableListInfo); for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); + QUERY_CHECK_NULL(pKeyInfo, code, lino, _end, terrno); void* tmp = taosArrayPush(pUidList, &pKeyInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } @@ -1501,7 +1531,7 @@ _end: return pUidList; } -static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { +static int32_t extractTableList(SArray* pList, const SOperatorInfo* pOperator) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1509,23 +1539,25 @@ static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pScanInfo = pOperator->info; STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; - void* tmp = taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); + + void* tmp = taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = pOperator->info; - void* tmp = taosArrayPush(pList, &pScanInfo->base.pTableListInfo); + + void* tmp = taosArrayPush(pList, &pScanInfo->base.pTableListInfo); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } else { if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) { - extractTableList(pList, pOperator->pDownstream[0]); + code = extractTableList(pList, pOperator->pDownstream[0]); } } _end: if (code != TSDB_CODE_SUCCESS) { - qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); + qError("%s %s failed at line %d since %s", pTaskInfo->id.str, __func__, lino, tstrerror(code)); } + return code; } int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList) { @@ -1533,16 +1565,17 @@ int32_t getTableListInfo(const SExecTaskInfo* pTaskInfo, SArray** pList) { return TSDB_CODE_INVALID_PARA; } + *pList = NULL; SArray* pArray = taosArrayInit(0, POINTER_BYTES); if (pArray == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } - SOperatorInfo* pOperator = pTaskInfo->pRoot; - extractTableList(pArray, pOperator); - - *pList = pArray; - return TSDB_CODE_SUCCESS; + int32_t code = extractTableList(pArray, pTaskInfo->pRoot); + if (code == 0) { + *pList = pArray; + } + return code; } int32_t qStreamOperatorReleaseState(qTaskInfo_t tInfo) { diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index e339ca5d29..cbb124b9e0 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -158,8 +158,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (isIntervalQuery) { if (p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists. pResult = getResultRowByPos(pResultBuf, p1, true); - if (NULL == pResult) { - T_LONG_JMP(pTaskInfo->env, terrno); + if (pResult == NULL) { + pTaskInfo->code = terrno; + return NULL; } ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); @@ -171,7 +172,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // todo pResult = getResultRowByPos(pResultBuf, p1, true); if (NULL == pResult) { - T_LONG_JMP(pTaskInfo->env, terrno); + pTaskInfo->code = terrno; + return NULL; } ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); @@ -184,7 +186,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); if (pPage == NULL) { qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); - T_LONG_JMP(pTaskInfo->env, terrno); + pTaskInfo->code = terrno; + return NULL; } releaseBufPage(pResultBuf, pPage); } @@ -193,7 +196,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResult == NULL) { pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize); if (pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, terrno); + pTaskInfo->code = terrno; + return NULL; } // add a new result set for a new group @@ -202,7 +206,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR sizeof(SResultRowPosition)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); + pTaskInfo->code = code; + return NULL; } } @@ -212,7 +217,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // too many time window in query if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH && tSimpleHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + pTaskInfo->code = TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW; + return NULL; } return pResult; @@ -583,8 +589,10 @@ int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* p size_t size = taosArrayGetSize(pColMatchInfo->pList); for (int32_t i = 0; i < size; ++i) { SColMatchItem* pInfo = taosArrayGet(pColMatchInfo->pList, i); + QUERY_CHECK_NULL(pInfo, code, lino, _err, terrno); if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->dstSlotId); + QUERY_CHECK_NULL(pColData, code, lino, _err, terrno); if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { code = blockDataUpdateTsWindow(pBlock, pInfo->dstSlotId); QUERY_CHECK_CODE(code, lino, _err); @@ -673,6 +681,7 @@ void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultR // expand the result into multiple rows. E.g., _wstart, top(k, 20) // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); @@ -1052,6 +1061,10 @@ bool groupbyTbname(SNodeList* pGroupList) { bool bytbname = false; if (LIST_LENGTH(pGroupList) == 1) { SNode* p = nodesListGetNode(pGroupList, 0); + if (!p) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return false; + } if (p->type == QUERY_NODE_FUNCTION) { // partition by tbname/group by tbname bytbname = (strcmp(((struct SFunctionNode*)p)->functionName, "tbname") == 0); @@ -1100,6 +1113,11 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i); + if (!pTable) { + taosArrayDestroy(pDeleterParam->pUidList); + taosMemoryFree(pDeleterParam); + return TSDB_CODE_OUT_OF_MEMORY; + } void* tmp = taosArrayPush(pDeleterParam->pUidList, &pTable->uid); if (!tmp) { taosArrayDestroy(pDeleterParam->pUidList); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 50117f6c7e..f61b514626 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -58,7 +58,7 @@ typedef struct SFillOperatorInfo { static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, int32_t order); static void destroyFillOperatorInfo(void* param); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); -static void fillResetPrevForNewGroup(SFillInfo* pFillInfo); +static int32_t fillResetPrevForNewGroup(SFillInfo* pFillInfo); static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, @@ -83,7 +83,11 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes); if (pInfo->pFillInfo->type == TSDB_FILL_PREV || pInfo->pFillInfo->type == TSDB_FILL_LINEAR) { - fillResetPrevForNewGroup(pInfo->pFillInfo); + int32_t code = fillResetPrevForNewGroup(pInfo->pFillInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } } int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows; @@ -146,13 +150,22 @@ _end: } } -static void fillResetPrevForNewGroup(SFillInfo* pFillInfo) { +static int32_t fillResetPrevForNewGroup(SFillInfo* pFillInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; for (int32_t colIdx = 0; colIdx < pFillInfo->numOfCols; ++colIdx) { if (!pFillInfo->pFillCol[colIdx].notFillCol) { SGroupKeys* key = taosArrayGet(pFillInfo->prev.pRowVal, colIdx); + QUERY_CHECK_NULL(key, code, lino, _end, terrno); key->isNull = true; } } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } // todo refactor: decide the start key according to the query time range. @@ -554,7 +567,10 @@ _error: } pTaskInfo->code = code; - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index c753f0fd9b..00b8c3b9ae 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -1504,7 +1504,10 @@ _error: destroyGroupCacheOperator(pInfo); } - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 502f9ff0f7..69a9045004 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -100,6 +100,10 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = (SColumn*)taosArrayGet(pGroupColList, i); + if (!pCol) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; @@ -611,8 +615,12 @@ _error: if (pInfo != NULL) { destroyGroupOperatorInfo(pInfo); } - destroyOperator(pOperator); - taosMemoryFreeClear(pOperator); + + if (pOperator) { + pOperator->info = NULL; + destroyOperator(pOperator); + } + return code; } @@ -888,10 +896,10 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { int32_t size = taosArrayGetSize(pInfo->sortedGroupArray); for (int32_t i = 0; i < size; i++) { SDataGroupInfo* pGp = taosArrayGet(pInfo->sortedGroupArray, i); - if (pGp->blockForNotLoaded) { + if (pGp && pGp->blockForNotLoaded) { for (int32_t i = 0; i < pGp->blockForNotLoaded->size; i++) { SSDataBlock** pBlock = taosArrayGet(pGp->blockForNotLoaded, i); - blockDataDestroy(*pBlock); + if (pBlock) blockDataDestroy(*pBlock); } taosArrayClear(pGp->blockForNotLoaded); pGp->offsetForNotLoaded = 0; @@ -916,6 +924,9 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { static SSDataBlock* buildPartitionResultForNotLoadBlock(SDataGroupInfo* pGroupInfo) { if (pGroupInfo->blockForNotLoaded && pGroupInfo->offsetForNotLoaded < pGroupInfo->blockForNotLoaded->size) { SSDataBlock** pBlock = taosArrayGet(pGroupInfo->blockForNotLoaded, pGroupInfo->offsetForNotLoaded); + if (!pBlock) { + return NULL; + } pGroupInfo->offsetForNotLoaded++; return *pBlock; } @@ -946,10 +957,18 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { ++pInfo->groupIndex; pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex); + if (pGroupInfo == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } pInfo->pageIndex = 0; } int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); + if (pageId == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } void* page = getBufPage(pInfo->pBuf, *pageId); if (page == NULL) { qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); @@ -1104,7 +1123,9 @@ static void destroyPartitionOperatorInfo(void* param) { int32_t size = taosArrayGetSize(pInfo->sortedGroupArray); for (int32_t i = 0; i < size; i++) { SDataGroupInfo* pGp = taosArrayGet(pInfo->sortedGroupArray, i); - taosArrayDestroy(pGp->pPageList); + if (pGp) { + taosArrayDestroy(pGp->pPageList); + } } taosArrayDestroy(pInfo->sortedGroupArray); @@ -1139,8 +1160,8 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo int32_t numOfCols = 0; SExprInfo* pExprInfo = NULL; - code = createExprInfo(pPartNode->pTargets, NULL, &pExprInfo, &numOfCols); + QUERY_CHECK_CODE(code, lino, _error); pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys); @@ -1225,7 +1246,10 @@ _error: destroyPartitionOperatorInfo(pInfo); } pTaskInfo->code = code; - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } TAOS_RETURN(code); } @@ -1237,6 +1261,9 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); + if (pResultRow == NULL || pTaskInfo->code != 0) { + return pTaskInfo->code; + } return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); } @@ -1276,7 +1303,9 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) { int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId); + QUERY_CHECK_NULL(pSrcCol, code, lino, _end, terrno); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j); + QUERY_CHECK_NULL(pDestCol, code, lino, _end, terrno); bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); char* pSrcData = NULL; if (!isNull) pSrcData = colDataGetData(pSrcCol, rowIndex); @@ -1342,6 +1371,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag QUERY_CHECK_CODE(code, lino, _end); SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + QUERY_CHECK_NULL(pTbCol, code, lino, _end, terrno); memset(tbName, 0, TSDB_TABLE_NAME_LEN); int32_t len = 0; if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { @@ -1358,6 +1388,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag pDestBlock->info.rows--; } else { void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + QUERY_CHECK_NULL(pTbNameCol, code, lino, _end, terrno); colDataSetNULL(pTbNameCol, pDestBlock->info.rows); tbName[0] = 0; } @@ -1371,6 +1402,7 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag } void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + QUERY_CHECK_NULL(pGpIdCol, code, lino, _end, terrno); code = colDataSetVal(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); QUERY_CHECK_CODE(code, lino, _end); pDestBlock->info.rows++; @@ -1557,7 +1589,11 @@ static void destroyStreamPartitionOperatorInfo(void* param) { taosArrayDestroy(pInfo->partitionSup.pGroupCols); for (int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++) { - SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i); + void* tmp = taosArrayGet(pInfo->partitionSup.pGroupColVals, i); + if (!tmp) { + continue; + } + SGroupKeys key = *(SGroupKeys*)tmp; taosMemoryFree(key.pData); } taosArrayDestroy(pInfo->partitionSup.pGroupColVals); @@ -1758,8 +1794,11 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart _error: pTaskInfo->code = code; - destroyStreamPartitionOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamPartitionOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return code; } @@ -1777,6 +1816,7 @@ int32_t extractColumnInfo(SNodeList* pNodeList, SArray** pArrayRes) { for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); + QUERY_CHECK_NULL(pNode, code, lino, _end, terrno); if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index 3b390c8719..8bc7c2db50 100644 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -594,9 +594,11 @@ int32_t createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numS SSortMergeInfo* pSortMergeInfo = &pInfo->sortMergeInfo; initLimitInfo(pMergePhyNode->node.pLimit, pMergePhyNode->node.pSlimit, &pInfo->limitInfo); pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + TSDB_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc); + TSDB_CHECK_NULL(pInputBlock, code, lino, _error, terrno); initResultSizeInfo(&pOperator->resultInfo, 1024); code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -620,6 +622,8 @@ int32_t createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numS case MERGE_TYPE_NON_SORT: { SNonSortMergeInfo* pNonSortMerge = &pInfo->nsortMergeInfo; pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + TSDB_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); + initResultSizeInfo(&pOperator->resultInfo, 1024); code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); TSDB_CHECK_CODE(code, lino, _error); @@ -629,6 +633,8 @@ int32_t createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size_t numS case MERGE_TYPE_COLUMNS: { SColsMergeInfo* pColsMerge = &pInfo->colsMergeInfo; pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + TSDB_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); + initResultSizeInfo(&pOperator->resultInfo, 1); code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); TSDB_CHECK_CODE(code, lino, _error); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 701ed0ddbc..af52c31364 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -295,6 +295,11 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo); if (code) { @@ -330,6 +335,11 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); @@ -362,6 +372,11 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (pHandle->vnode) { code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, @@ -376,6 +391,11 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand //pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); code = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, &pOperator); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + return code; + } } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; code = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo, &pOperator); @@ -385,19 +405,36 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pTagScanPhyNode = (STagScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (!pTagScanPhyNode->onlyMetaCtbIdx) { code = createScanTableListInfo((SScanPhysiNode*)pTagScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; qError("failed to getTableList, code: %s", tstrerror(code)); + tableListDestroy(pTableListInfo); return code; } } - code = createTagScanOperatorInfo(pHandle, pTagScanPhyNode, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo, &pOperator); + code = createTagScanOperatorInfo(pHandle, pTagScanPhyNode, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo, + &pOperator); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + return code; + } } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (pBlockNode->tableType == TSDB_SUPER_TABLE) { SArray* pList = taosArrayInit(4, sizeof(uint64_t)); @@ -405,6 +442,7 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; taosArrayDestroy(pList); + tableListDestroy(pTableListInfo); return code; } @@ -412,13 +450,14 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand for (int32_t i = 0; i < num; ++i) { uint64_t* id = taosArrayGet(pList, i); if (id == NULL) { - pTaskInfo->code = terrno; - return terrno; + continue; } code = tableListAddTableInfo(pTableListInfo, *id, 0); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + taosArrayDestroy(pList); return code; } } @@ -428,29 +467,47 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand code = tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0); if (code) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); return code; } } code = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo, &pOperator); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + return code; + } } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; STableListInfo* pTableListInfo = tableListCreate(); + if (!pTableListInfo) { + pTaskInfo->code = terrno; + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } - code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - pTagCond, pTagIndexCond, pTaskInfo); + code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, pTagCond, + pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); return code; } code = initQueriedTableSchemaInfo(pHandle, &pScanNode->scan, dbname, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; + tableListDestroy(pTableListInfo); return code; } code = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo, &pOperator); + if (code) { + tableListDestroy(pTableListInfo); + pTaskInfo->code = code; + return code; + } } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo, &pOperator); } else { @@ -593,7 +650,7 @@ void destroyOperator(SOperatorInfo* pOperator) { freeResetOperatorParams(pOperator, OP_GET_PARAM, true); freeResetOperatorParams(pOperator, OP_NOTIFY_PARAM, true); - if (pOperator->fpSet.closeFn != NULL) { + if (pOperator->fpSet.closeFn != NULL && pOperator->info != NULL) { pOperator->fpSet.closeFn(pOperator->info); } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 7185f74254..3f1eb43578 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -180,7 +180,10 @@ int32_t createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* _error: destroyProjectOperatorInfo(pInfo); - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -485,6 +488,7 @@ int32_t createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* } SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->node.pOutputDataBlockDesc); + TSDB_CHECK_NULL(pResBlock, code, lino, _error, terrno); // Make sure the size of SSDataBlock will never exceed the size of 2MB. int32_t TWOMB = 2 * 1024 * 1024; @@ -528,7 +532,10 @@ int32_t createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* _error: destroyIndefinitOperatorInfo(pInfo); - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -702,6 +709,9 @@ int32_t setFunctionResultOutput(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, int64_t groupId = 0; SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, (char*)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup, true); + if (pRow == NULL || pTaskInfo->code != 0) { + return pTaskInfo->code; + } for (int32_t i = 0; i < numOfExprs; ++i) { struct SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, rowEntryInfoOffset); diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 9f4d9c4405..7100e10276 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -64,6 +64,10 @@ int32_t doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC p->id.queryId = queryId; p->id.taskId = taskId; p->id.str = taosMemoryMalloc(64); + if (p->id.str == NULL) { + return terrno; + } + buildTaskId(taskId, queryId, p->id.str); p->schemaInfos = taosArrayInit(1, sizeof(SSchemaInfo)); if (p->id.str == NULL || p->schemaInfos == NULL) { @@ -132,8 +136,7 @@ void cleanupQueriedTableScanInfo(void* p) { int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo) { SMetaReader mr = {0}; if (pHandle == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return TSDB_CODE_INVALID_PARA; } SStorageAPI* pAPI = &pTaskInfo->storageAPI; @@ -145,13 +148,18 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo GET_TASKID(pTaskInfo)); pAPI->metaReaderFn.clearReader(&mr); - return terrno; + return code; } SSchemaInfo schemaInfo = {0}; schemaInfo.tablename = taosStrdup(mr.me.name); schemaInfo.dbname = taosStrdup(dbName); + if (schemaInfo.tablename == NULL || schemaInfo.dbname == NULL) { + pAPI->metaReaderFn.clearReader(&mr); + cleanupQueriedTableScanInfo(&schemaInfo); + return terrno; + } if (mr.me.type == TSDB_SUPER_TABLE) { schemaInfo.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); @@ -163,9 +171,8 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo code = pAPI->metaReaderFn.getEntryGetUidCache(&mr, suid); if (code != TSDB_CODE_SUCCESS) { pAPI->metaReaderFn.clearReader(&mr); - taosMemoryFree(schemaInfo.tablename); - taosMemoryFree(schemaInfo.dbname); - return terrno; + cleanupQueriedTableScanInfo(&schemaInfo); + return code; } schemaInfo.sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); @@ -175,10 +182,25 @@ int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNo } pAPI->metaReaderFn.clearReader(&mr); + + if (schemaInfo.sw == NULL) { + cleanupQueriedTableScanInfo(&schemaInfo); + return terrno; + } + schemaInfo.qsw = extractQueriedColumnSchema(pScanNode); - + if (schemaInfo.qsw == NULL) { + cleanupQueriedTableScanInfo(&schemaInfo); + return terrno; + } + void* p = taosArrayPush(pTaskInfo->schemaInfos, &schemaInfo); - return (p != NULL)? TSDB_CODE_SUCCESS:TSDB_CODE_OUT_OF_MEMORY; + if (p == NULL) { + cleanupQueriedTableScanInfo(&schemaInfo); + return terrno; + } + + return code; } SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { @@ -186,7 +208,14 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols); SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pqSw == NULL) { + return NULL; + } + pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema)); + if (pqSw->pSchema == NULL) { + return NULL; + } for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 37c5bc1aeb..647909cc13 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -88,20 +88,26 @@ static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } -static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order) { +static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order, bool* overlap) { + int32_t code = TSDB_CODE_SUCCESS; STimeWindow w = {0}; // 0 by default, which means it is not a interval operator of the upstream operator. if (pInterval->interval == 0) { - return false; + *overlap = false; + return code; } if (order == TSDB_ORDER_ASC) { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey); - ASSERT(w.ekey >= pBlockInfo->window.skey); + if(w.ekey < pBlockInfo->window.skey) { + qError("w.ekey:%" PRId64 " < pBlockInfo->window.skey:%" PRId64, w.ekey, pBlockInfo->window.skey); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } if (w.ekey < pBlockInfo->window.ekey) { - return true; + *overlap = true; + return code; } while (1) { @@ -110,17 +116,25 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn break; } - ASSERT(w.ekey > pBlockInfo->window.ekey); + if(w.ekey <= pBlockInfo->window.ekey) { + qError("w.ekey:%" PRId64 " <= pBlockInfo->window.ekey:%" PRId64, w.ekey, pBlockInfo->window.ekey); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) { - return true; + *overlap = true; + return code; } } } else { w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey); - ASSERT(w.skey <= pBlockInfo->window.ekey); + if(w.skey > pBlockInfo->window.ekey) { + qError("w.skey:%" PRId64 " > pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.ekey); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } if (w.skey > pBlockInfo->window.skey) { - return true; + *overlap = true; + return code; } while (1) { @@ -129,14 +143,19 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn break; } - ASSERT(w.skey < pBlockInfo->window.skey); + if(w.skey >= pBlockInfo->window.skey){ + qError("w.skey:%" PRId64 " >= pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.skey); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } if (pBlockInfo->window.skey <= TMIN(w.ekey, pBlockInfo->window.ekey)) { - return true; + *overlap = true; + return code; } } } - return false; + *overlap = false; + return code; } // this function is for table scanner to extract temporary results of upstream aggregate results. @@ -319,9 +338,18 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca bool loadSMA = false; *status = pTableScanInfo->dataBlockLoadFlag; - if (pOperator->exprSupp.pFilterInfo != NULL || - overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order)) { + if (pOperator->exprSupp.pFilterInfo != NULL) { (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; + } else { + bool overlap = false; + int ret = + overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order, &overlap); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + if (overlap) { + (*status) = FUNC_DATA_REQUIRED_DATA_LOAD; + } } SDataBlockInfo* pBlockInfo = &pBlock->info; @@ -358,7 +386,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } } - ASSERT(*status == FUNC_DATA_REQUIRED_DATA_LOAD); + if(*status != FUNC_DATA_REQUIRED_DATA_LOAD) { + qError("[loadDataBlock] invalid status:%d", *status); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } // try to filter data block according to sma info if (pOperator->exprSupp.pFilterInfo != NULL && (!loadSMA)) { @@ -413,7 +444,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca return code; } - ASSERT(p == pBlock); + if(p != pBlock) { + qError("[loadDataBlock] p != pBlock"); + return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR; + } doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -1071,6 +1105,10 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) { int32_t tableIdx = 0; for (int32_t i = 0; i < num; ++i) { uint64_t* pUid = taosArrayGet(pParam->pUidList, i); + if (!pUid) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (taosHashPut(pListInfo->map, pUid, sizeof(uint64_t), &tableIdx, sizeof(int32_t))) { if (TSDB_CODE_DUP_KEY == terrno) { @@ -1257,8 +1295,14 @@ static int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { (*ppRes) = NULL; return code; } - - tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + STableKeyInfo* tmp = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); + if (!tmp) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + taosRUnLockLatch(&pTaskInfo->lock); + (*ppRes) = NULL; + return terrno; + } + tInfo = *tmp; taosRUnLockLatch(&pTaskInfo->lock); code = pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); @@ -1418,10 +1462,14 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa _error: if (pInfo != NULL) { + pInfo->base.pTableListInfo = NULL; // this attribute will be destroy outside of this function destroyTableScanOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -3178,6 +3226,8 @@ FETCH_NEXT_BLOCK: qDebug("process %d/%d input data blocks, %s", current, (int32_t)total, id); SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + QUERY_CHECK_NULL(pPacked, code, lino, _end, terrno); + SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.parTbName[0]) { code = @@ -3386,6 +3436,7 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + QUERY_CHECK_NULL(pSubmit, code, lino, _end, terrno); qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id); if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < @@ -3473,7 +3524,9 @@ FETCH_NEXT_BLOCK: qDebug("process %d/%d input data blocks, %s", current, (int32_t)total, id); SPackedData* pData = taosArrayGet(pInfo->pBlockLists, current); + QUERY_CHECK_NULL(pData, code, lino, _end, terrno); SSDataBlock* pBlock = taosArrayGet(pData->pDataBlock, 0); + QUERY_CHECK_NULL(pBlock, code, lino, _end, terrno); if (pBlock->info.type == STREAM_CHECKPOINT) { streamScanOperatorSaveCheckpoint(pInfo); @@ -3507,6 +3560,7 @@ static int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** size_t size = tableListGetSize(pTableListInfo); for (int32_t i = 0; i < size; ++i) { STableKeyInfo* pkeyInfo = tableListGetInfo(pTableListInfo, i); + QUERY_CHECK_NULL(pkeyInfo, code, lino, _end, terrno); void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid); QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); } @@ -3685,15 +3739,18 @@ int32_t createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo QRY_OPTR_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SStreamRawScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamRawScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + lino = __LINE__; goto _end; } pInfo->pTableListInfo = tableListCreate(); + QUERY_CHECK_NULL(pInfo->pTableListInfo, code, lino, _end, terrno); pInfo->vnode = pHandle->vnode; pInfo->pAPI = &pTaskInfo->storageAPI; @@ -3707,6 +3764,9 @@ int32_t createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo return code; _end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = code; @@ -3720,7 +3780,7 @@ static void destroyStreamScanOperatorInfo(void* param) { destroyOperator(pStreamScan->pTableScanOp); } - if (pStreamScan->tqReader) { + if (pStreamScan->tqReader != NULL && pStreamScan->readerFn.tqReaderClose != NULL) { pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader); } if (pStreamScan->matchInfo.pList) { @@ -3876,7 +3936,6 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); goto _error; } @@ -3889,7 +3948,6 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* int32_t numOfCols = 0; code = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { - tableListDestroy(pTableListInfo); goto _error; } @@ -3901,6 +3959,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* for (int32_t i = 0; i < numOfOutput; ++i) { SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i); + QUERY_CHECK_NULL(id, code, lino, _error, terrno); int16_t colId = id->colId; void* tmp = taosArrayPush(pColIds, &colId); @@ -3919,8 +3978,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* if (pTableScanNode->pSubtable != NULL) { SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); + code = terrno; goto _error; } @@ -3929,7 +3987,6 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* QUERY_CHECK_CODE(code, lino, _error); if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) { - tableListDestroy(pTableListInfo); goto _error; } } @@ -3939,20 +3996,17 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); goto _error; } if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } } pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tableListDestroy(pTableListInfo); + code = terrno; goto _error; } @@ -4096,7 +4150,10 @@ _error: destroyStreamScanOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -4110,6 +4167,14 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); + if (!item) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + tDecoderClear(&(*mr).coder); + pAPI->metaReaderFn.clearReader(mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid); tDecoderClear(&(*mr).coder); if (code != TSDB_CODE_SUCCESS) { @@ -4254,11 +4319,12 @@ static int32_t tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; int32_t numOfTables = taosArrayGetSize(aUidTags); + SArray* pBlockList = NULL; SSDataBlock* pResBlock = createTagValBlockForFilter(pInfo->filterCtx.cInfoList, numOfTables, aUidTags, pVnode, pAPI); QUERY_CHECK_NULL(pResBlock, code, lino, _end, terrno); - SArray* pBlockList = taosArrayInit(1, POINTER_BYTES); + pBlockList = taosArrayInit(1, POINTER_BYTES); QUERY_CHECK_NULL(pBlockList, code, lino, _end, terrno); void* tmp = taosArrayPush(pBlockList, &pResBlock); @@ -4359,8 +4425,10 @@ static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRe for (int i = 0; i < szTables; ++i) { int32_t idx = *(int32_t*)taosArrayGet(aFilterIdxs, i); STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx); + QUERY_CHECK_NULL(pUidTagInfo, code, lino, _end, terrno); for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); + QUERY_CHECK_NULL(pDst, code, lino, _end, terrno); code = tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode); } } @@ -4368,8 +4436,10 @@ static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRe size_t szTables = taosArrayGetSize(aUidTags); for (int i = 0; i < szTables; ++i) { STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i); + QUERY_CHECK_NULL(pUidTagInfo, code, lino, _end, terrno); for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); + QUERY_CHECK_NULL(pDst, code, lino, _end, terrno); code = tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode); } } @@ -4544,7 +4614,7 @@ static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) { static void destroyTagScanOperatorInfo(void* param) { STagScanInfo* pInfo = (STagScanInfo*)param; - if (pInfo->pCtbCursor != NULL) { + if (pInfo->pCtbCursor != NULL && pInfo->pStorageAPI != NULL) { pInfo->pStorageAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor); } taosHashCleanup(pInfo->filterCtx.colHash); @@ -4636,8 +4706,15 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p return code; _error: - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); + if (pInfo) { + pInfo->pTableListInfo = NULL; + } + + if (pInfo != NULL) destroyTagScanOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } @@ -4787,18 +4864,23 @@ static int32_t fetchNextSubTableBlockFromReader(SOperatorInfo* pOperator, STmsSu return TSDB_CODE_SUCCESS; } -static void setGroupStartEndIndex(STableMergeScanInfo* pInfo) { +static int32_t setGroupStartEndIndex(STableMergeScanInfo* pInfo) { pInfo->bGroupProcessed = false; size_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); + if (!tableKeyInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (tableKeyInfo->groupId != pInfo->groupId) { break; } } pInfo->tableEndIndex = i - 1; + return TSDB_CODE_SUCCESS; } static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { @@ -4815,6 +4897,10 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock; SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId); + if (!col) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } pInput->aTs = (int64_t*)col->pData; } @@ -4823,14 +4909,26 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { } static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { - setGroupStartEndIndex(pInfo); + int32_t code = setGroupStartEndIndex(pInfo); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + return code; + } STmsSubTablesMergeInfo* pSubTblsInfo = taosMemoryCalloc(1, sizeof(STmsSubTablesMergeInfo)); if (pSubTblsInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pSubTblsInfo->pTsOrderInfo = taosArrayGet(pInfo->pSortInfo, 0); + if (!pSubTblsInfo->pTsOrderInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } if (taosArrayGetSize(pInfo->pSortInfo) == 2) { pSubTblsInfo->pPkOrderInfo = taosArrayGet(pInfo->pSortInfo, 1); + if (!pSubTblsInfo->pPkOrderInfo) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } } else { pSubTblsInfo->pPkOrderInfo = NULL; } @@ -4842,7 +4940,7 @@ static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { } int32_t bufPageSize = pInfo->bufPageSize; int32_t inMemSize = (pSubTblsInfo->numSubTables - pSubTblsInfo->numTableBlocksInMem) * bufPageSize; - int32_t code = + code = createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pSubTblsInfo->aInputs); @@ -4952,6 +5050,10 @@ static int32_t adjustSubTableForNextRow(SOperatorInfo* pOperatorInfo, STmsSubTab } if (pInput->rowIdx != -1) { SColumnInfoData* col = taosArrayGet(pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId); + if (!col) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + return terrno; + } pInput->pInputBlock = pInputBlock; pInput->aTs = (int64_t*)col->pData; } @@ -4970,8 +5072,10 @@ static int32_t appendChosenRowToDataBlock(STmsSubTablesMergeInfo* pSubTblsInfo, for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno); SColumnInfoData* pSrcColInfo = taosArrayGet(pInputBlock->pDataBlock, i); + QUERY_CHECK_NULL(pSrcColInfo, code, lino, _end, terrno); bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL); if (isNull) { @@ -5114,7 +5218,9 @@ int32_t doTableMergeScanParaSubTablesNext(SOperatorInfo* pOperator, SSDataBlock* return code; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex))->groupId; + STableKeyInfo* pTmpGpId = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(pTmpGpId, code, lino, _end, terrno); + pInfo->groupId = pTmpGpId->groupId; code = startSubTablesTableMergeScan(pOperator); QUERY_CHECK_CODE(code, lino, _end); } @@ -5128,6 +5234,7 @@ int32_t doTableMergeScanParaSubTablesNext(SOperatorInfo* pOperator, SSDataBlock* pBlock = getSubTablesSortedBlock(pOperator, pInfo->pResBlock, pOperator->resultInfo.capacity); if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) { STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(tbInfo, code, lino, _end, terrno); pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo); } if (pBlock != NULL) { @@ -5144,7 +5251,10 @@ int32_t doTableMergeScanParaSubTablesNext(SOperatorInfo* pOperator, SSDataBlock* } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; + STableKeyInfo* pTmpGpId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(pTmpGpId, code, lino, _end, terrno); + + pInfo->groupId = pTmpGpId->groupId; code = startSubTablesTableMergeScan(pOperator); QUERY_CHECK_CODE(code, lino, _end); resetLimitInfoForNextGroup(&pInfo->limitInfo); @@ -5338,6 +5448,7 @@ int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppS int32_t pkTargetSlotId = -1; for (int32_t i = 0; i < taosArrayGetSize(colMatchInfo); ++i) { SColMatchItem* colInfo = taosArrayGet(colMatchInfo, i); + QUERY_CHECK_NULL(colInfo, code, lino, _end, terrno); if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tsTargetSlotId = colInfo->dstSlotId; biTs.order = order; @@ -5473,6 +5584,7 @@ void startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t i = pInfo->tableStartIndex + 1; for (; i < numOfTables; ++i) { STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i); + QUERY_CHECK_NULL(tableKeyInfo, code, lino, _end, terrno); if (tableKeyInfo->groupId != pInfo->groupId) { break; } @@ -5598,7 +5710,9 @@ int32_t doTableMergeScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { return code; } pInfo->tableStartIndex = 0; - pInfo->groupId = ((STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex))->groupId; + STableKeyInfo* tmp = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); + pInfo->groupId = tmp->groupId; startGroupTableMergeScan(pOperator); } @@ -5612,6 +5726,7 @@ int32_t doTableMergeScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { pOperator); if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) { STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(tbInfo, code, lino, _end, terrno); pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo); } if (pBlock != NULL) { @@ -5633,7 +5748,9 @@ int32_t doTableMergeScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { } pInfo->tableStartIndex = pInfo->tableEndIndex + 1; - pInfo->groupId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex)->groupId; + STableKeyInfo* tmp = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex); + QUERY_CHECK_NULL(tmp, code, lino, _end, terrno); + pInfo->groupId = tmp->groupId; startGroupTableMergeScan(pOperator); resetLimitInfoForNextGroup(&pInfo->limitInfo); } @@ -5662,8 +5779,10 @@ void destroyTableMergeScanOperatorInfo(void* param) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; // start one reader variable - pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); - pTableScanInfo->base.dataReader = NULL; + if (pTableScanInfo->base.readerAPI.tsdReaderClose != NULL) { + pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); + pTableScanInfo->base.dataReader = NULL; + } for (int32_t i = 0; i < pTableScanInfo->numNextDurationBlocks; ++i) { if (pTableScanInfo->nextDurationBlocks[i] != NULL) { @@ -5718,7 +5837,8 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR SOperatorInfo** pOptrInfo) { QRY_OPTR_CHECK(pOptrInfo); - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5731,16 +5851,10 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR int32_t numOfCols = 0; code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); - int32_t lino = 0; - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pInfo->base.matchInfo.pList); - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); if (pTableScanNode->scan.pScanPseudoCols != NULL) { SExprSupp* pSup = &pInfo->base.pseudoSup; @@ -5755,10 +5869,7 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5); - if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) { - code = terrno; - goto _error; - } + QUERY_CHECK_NULL(pInfo->base.metaCache.pTableMetaEntryCache, code, lino, _error, terrno); pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader; pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; @@ -5775,9 +5886,7 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR pInfo->sample.seed = taosGetTimestampSec(); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); @@ -5839,9 +5948,16 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR return code; _error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + pTaskInfo->code = code; + pInfo->base.pTableListInfo = NULL; + if (pInfo != NULL) destroyTableMergeScanOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } @@ -5998,7 +6114,10 @@ _error: if (pInfo != NULL) { destoryTableCountScanOperator(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -6010,6 +6129,7 @@ int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, ch if (pSupp->dbNameSlotId != -1) { ASSERT(strlen(dbName)); SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); + QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno); char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; tstrncpy(varDataVal(varDbName), dbName, TSDB_DB_NAME_LEN); @@ -6021,6 +6141,7 @@ int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, ch if (pSupp->stbNameSlotId != -1) { SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); + QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno); if (strlen(stbName) != 0) { char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; strncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN); @@ -6034,6 +6155,7 @@ int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, ch if (pSupp->tbCountSlotId != -1) { SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId); + QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno); code = colDataSetVal(colInfoData, 0, (char*)&count, false); QUERY_CHECK_CODE(code, lino, _end); } diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 858f26ad18..59b4e1cbbb 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -101,7 +101,10 @@ int32_t createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortN } pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + QUERY_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); + pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); + TSDB_CHECK_NULL(pInfo->pSortInfo, code, lino, _error, terrno); if (pSortNode->calcGroupId) { int32_t keyLen; @@ -120,13 +123,14 @@ int32_t createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortN if (TSDB_CODE_SUCCESS == code) { // PK ts col should always at last, see partColOptCreateSort if (pSortNode->excludePkCol) taosArrayPop(pGroupIdCalc->pSortColsArr); - keyLen = extractKeysLen(pGroupIdCalc->pSortColsArr); + code = extractKeysLen(pGroupIdCalc->pSortColsArr, &keyLen); + QUERY_CHECK_CODE(code, lino, _error); } if (TSDB_CODE_SUCCESS == code) { pGroupIdCalc->lastKeysLen = 0; pGroupIdCalc->keyBuf = taosMemoryCalloc(1, keyLen); if (!pGroupIdCalc->keyBuf) { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } } } @@ -160,7 +164,10 @@ _error: destroySortOperatorInfo(pInfo); } - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -366,8 +373,13 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + if (ps == NULL) { + return terrno; + } + ps->param = pOperator->pDownstream[0]; ps->onlyRef = true; + code = tsortAddSource(pInfo->pSortHandle, ps); if (code) { taosMemoryFree(ps); @@ -460,6 +472,9 @@ void destroySortOperatorInfo(void* param) { int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { SSortExecInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortExecInfo)); + if (pInfo == NULL) { + return terrno; + } SSortOperatorInfo* pOperatorInfo = (SSortOperatorInfo*)pOptr->info; @@ -634,6 +649,10 @@ int32_t beginSortGroup(SOperatorInfo* pOperator) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); SGroupSortSourceParam* param = taosMemoryCalloc(1, sizeof(SGroupSortSourceParam)); + if (ps == NULL || param == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + param->childOpInfo = pOperator->pDownstream[0]; param->grpSortOpInfo = pInfo; ps->param = param; @@ -788,6 +807,8 @@ int32_t createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNo QUERY_CHECK_NULL(pOperator->exprSupp.pCtx, code, lino, _error, terrno); pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); + QUERY_CHECK_NULL(pInfo->binfo.pRes, code, lino, _error, terrno); + code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); TSDB_CHECK_CODE(code, lino, _error); @@ -818,6 +839,9 @@ _error: if (pInfo != NULL) { destroyGroupSortOperatorInfo(pInfo); } - taosMemoryFree(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 499c08f89d..62506858fc 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -926,7 +926,10 @@ _error: destroyStreamCountAggOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return code; diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 54fddf843b..bde6198709 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -899,7 +899,7 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* QUERY_CHECK_CODE(code, lino, _error); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); - QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); + QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); QUERY_CHECK_CODE(code, lino, _error); @@ -980,8 +980,11 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* return code; _error: - destroyStreamEventOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamEventOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return code; diff --git a/source/libs/executor/src/streamfilloperator.c b/source/libs/executor/src/streamfilloperator.c index b7b5b1a38c..8d91af46b1 100644 --- a/source/libs/executor/src/streamfilloperator.c +++ b/source/libs/executor/src/streamfilloperator.c @@ -1458,8 +1458,11 @@ _error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); } - destroyStreamFillOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamFillOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index e95ffeb658..61d4eed156 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2012,8 +2012,11 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN return code; _error: - destroyStreamFinalIntervalOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -3738,9 +3741,7 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); pInfo->twAggSup = (STimeWindowAggSupp){ .waterMark = pSessionNode->window.watermark, @@ -3834,7 +3835,10 @@ _error: destroyStreamSessionAggOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return code; @@ -4090,7 +4094,10 @@ _error: if (pInfo != NULL) { destroyStreamSessionAggOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo)); @@ -4979,8 +4986,11 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* return code; _error: - destroyStreamStateOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamStateOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); return code; @@ -5193,6 +5203,8 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno); + initBasicInfo(&pInfo->binfo, pResBlock); + pInfo->interval = (SInterval){ .interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, @@ -5220,7 +5232,6 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* SExprSupp* pSup = &pOperator->exprSupp; pSup->hasWindowOrGroup = true; - initBasicInfo(&pInfo->binfo, pResBlock); code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); QUERY_CHECK_CODE(code, lino, _error); @@ -5314,8 +5325,11 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* return code; _error: - destroyStreamFinalIntervalOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 14fda685c0..e11ee6b0dc 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -568,7 +568,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); + (void)pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } if (pInfo->pSchema == NULL) { @@ -595,6 +595,9 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { void* schema = taosHashGet(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); if (schema == NULL) { SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); + if (pInfo->pCur->mr.me.stbEntry.schemaRow.pSchema) { + QUERY_CHECK_NULL(schemaWrapper, code, lino, _end, terrno); + } code = taosHashPut(pInfo->pSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); if (code == TSDB_CODE_DUP_KEY) { code = TSDB_CODE_SUCCESS; @@ -626,6 +629,9 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { return NULL; } SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&smrSuperTable.me.stbEntry.schemaRow); + if (smrSuperTable.me.stbEntry.schemaRow.pSchema) { + QUERY_CHECK_NULL(schemaWrapper, code, lino, _end, terrno); + } code = taosHashPut(pInfo->pSchema, &suid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); if (code == TSDB_CODE_DUP_KEY) { code = TSDB_CODE_SUCCESS; @@ -666,6 +672,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { } blockDataDestroy(pDataBlock); + pDataBlock = NULL; if (ret != 0) { pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; @@ -677,6 +684,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { _end: if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(pDataBlock); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); @@ -689,6 +697,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t lino = 0; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; + SSDataBlock* dataBlock = NULL; SSysTableScanInfo* pInfo = pOperator->info; if (pOperator->status == OP_EXEC_DONE) { @@ -698,7 +707,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); int32_t numOfRows = 0; - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); + dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); code = blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _end); @@ -771,8 +780,9 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { int32_t ret = 0; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + QUERY_CHECK_NULL(pInfo->pCur, code, lino, _end, terrno); } else { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); + (void)pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 0); } while ((ret = pAPI->metaFn.cursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { @@ -820,6 +830,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } blockDataDestroy(dataBlock); + dataBlock = NULL; if (ret != 0) { pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; @@ -831,6 +842,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + blockDataDestroy(dataBlock); pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); pInfo->pCur = NULL; pTaskInfo->code = code; @@ -985,16 +997,19 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, // table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, tableName, false); QUERY_CHECK_CODE(code, lino, _end); // database name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); // super table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, stableName, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1002,12 +1017,14 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, tagName, false); QUERY_CHECK_CODE(code, lino, _end); // tag type int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); char tagTypeStr[VARSTR_HEADER_SIZE + 32]; int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); if (tagType == TSDB_DATA_TYPE_NCHAR) { @@ -1073,6 +1090,7 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, } } pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, tagVarChar, (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); QUERY_CHECK_CODE(code, lino, _end); @@ -1108,15 +1126,18 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, // table name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, tName, false); QUERY_CHECK_CODE(code, lino, _end); // database name pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, tableType, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1124,12 +1145,14 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(colName, schemaRow->pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, colName, false); QUERY_CHECK_CODE(code, lino, _end); // col type int8_t colType = schemaRow->pSchema[i].type; pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); char colTypeStr[VARSTR_HEADER_SIZE + 32]; int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); if (colType == TSDB_DATA_TYPE_VARCHAR) { @@ -1144,11 +1167,13 @@ static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, QUERY_CHECK_CODE(code, lino, _end); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); QUERY_CHECK_CODE(code, lino, _end); for (int32_t j = 6; j <= 8; ++j) { pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, numOfRows); } ++numOfRows; @@ -1177,7 +1202,7 @@ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { } SSDataBlock* pBlock = NULL; - int32_t code = createDataBlock(&pBlock); + int32_t code = createDataBlock(&pBlock); if (code) { terrno = code; return NULL; @@ -1217,6 +1242,7 @@ int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTabl } SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); STR_TO_VARSTR(n, pm->name); code = colDataSetVal(pColInfoData, numOfRows, n, false); @@ -1225,26 +1251,31 @@ int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTabl // database name STR_TO_VARSTR(n, dbName); pColInfoData = taosArrayGet(p->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, numOfRows); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pm->colNum, false); QUERY_CHECK_CODE(code, lino, _end); for (int32_t j = 4; j <= 8; ++j) { pColInfoData = taosArrayGet(p->pDataBlock, j); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, numOfRows); } STR_TO_VARSTR(n, "SYSTEM_TABLE"); pColInfoData = taosArrayGet(p->pDataBlock, 9); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1285,9 +1316,11 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { QUERY_CHECK_CODE(code, lino, _end); blockDataDestroy(p); + p = NULL; _end: if (code != TSDB_CODE_SUCCESS) { + blockDataDestroy(p); qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } return code; @@ -1300,6 +1333,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { SStorageAPI* pAPI = &pTaskInfo->storageAPI; SSysTableScanInfo* pInfo = pOperator->info; SSysTableIndex* pIdx = pInfo->pIdx; + SSDataBlock* p = NULL; blockDataCleanup(pInfo->pRes); int32_t numOfRows = 0; @@ -1319,7 +1353,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { varDataSetLen(dbname, strlen(varDataVal(dbname))); - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); QUERY_CHECK_CODE(code, lino, _end); @@ -1327,6 +1361,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { int32_t i = pIdx->lastIdx; for (; i < taosArrayGetSize(pIdx->uids); i++) { tb_uid_t* uid = taosArrayGet(pIdx->uids, i); + QUERY_CHECK_NULL(uid, code, lino, _end, terrno); SMetaReader mr = {0}; pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn); @@ -1339,16 +1374,20 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // table name SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); + code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); // vgId pColInfoData = taosArrayGet(p->pDataBlock, 6); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1357,6 +1396,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // create time int64_t ts = mr.me.ctbEntry.btime; pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1373,18 +1413,21 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, terrno); } pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); QUERY_CHECK_CODE(code, lino, _end); // super table name STR_TO_VARSTR(n, mr1.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); pAPI->metaReaderFn.clearReader(&mr1); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); if (mr.me.ctbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); @@ -1401,11 +1444,13 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); QUERY_CHECK_CODE(code, lino, _end); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1414,20 +1459,24 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false); QUERY_CHECK_CODE(code, lino, _end); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); QUERY_CHECK_CODE(code, lino, _end); // super table name pColInfoData = taosArrayGet(p->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, numOfRows); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); if (mr.me.ntbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); @@ -1444,11 +1493,13 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.uid, false); QUERY_CHECK_CODE(code, lino, _end); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1459,6 +1510,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { pAPI->metaReaderFn.clearReader(&mr); pColInfoData = taosArrayGet(p->pDataBlock, 9); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1502,12 +1554,14 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { } blockDataDestroy(p); + p = NULL; pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + blockDataDestroy(p); pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } @@ -1520,14 +1574,16 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; int8_t firstMetaCursor = 0; + SSDataBlock* p = NULL; SSysTableScanInfo* pInfo = pOperator->info; if (pInfo->pCur == NULL) { pInfo->pCur = pAPI->metaFn.openTableMetaCursor(pInfo->readHandle.vnode); + QUERY_CHECK_NULL(pInfo->pCur, code, lino, _end, terrno); firstMetaCursor = 1; } if (!firstMetaCursor) { - pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 1); + (void)pAPI->metaFn.resumeTableMetaCursor(pInfo->pCur, 0, 1); } blockDataCleanup(pInfo->pRes); @@ -1547,7 +1603,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { varDataSetLen(dbname, strlen(varDataVal(dbname))); - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); QUERY_CHECK_NULL(p, code, lino, _end, terrno); code = blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); @@ -1561,16 +1617,19 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // table name SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); // database name pColInfoData = taosArrayGet(p->pDataBlock, 1); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, dbname, false); QUERY_CHECK_CODE(code, lino, _end); // vgId pColInfoData = taosArrayGet(p->pDataBlock, 6); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&vgId, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1579,6 +1638,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // create time int64_t ts = pInfo->pCur->mr.me.ctbEntry.btime; pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&ts, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1603,18 +1663,21 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); QUERY_CHECK_CODE(code, lino, _end); // super table name STR_TO_VARSTR(n, mr.me.name); pColInfoData = taosArrayGet(p->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); pAPI->metaReaderFn.clearReader(&mr); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); @@ -1631,11 +1694,13 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); QUERY_CHECK_CODE(code, lino, _end); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1643,20 +1708,24 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { } else if (tableType == TSDB_NORMAL_TABLE) { // create time pColInfoData = taosArrayGet(p->pDataBlock, 2); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.btime, false); QUERY_CHECK_CODE(code, lino, _end); // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); QUERY_CHECK_CODE(code, lino, _end); // super table name pColInfoData = taosArrayGet(p->pDataBlock, 4); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); colDataSetNULL(pColInfoData, numOfRows); // table comment pColInfoData = taosArrayGet(p->pDataBlock, 8); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); @@ -1673,11 +1742,13 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { // uid pColInfoData = taosArrayGet(p->pDataBlock, 5); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); QUERY_CHECK_CODE(code, lino, _end); // ttl pColInfoData = taosArrayGet(p->pDataBlock, 7); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1685,6 +1756,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { } pColInfoData = taosArrayGet(p->pDataBlock, 9); + QUERY_CHECK_NULL(pColInfoData, code, lino, _end, terrno); code = colDataSetVal(pColInfoData, numOfRows, n, false); QUERY_CHECK_CODE(code, lino, _end); @@ -1724,6 +1796,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { } blockDataDestroy(p); + p = NULL; // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found if (ret != 0) { @@ -1737,6 +1810,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { _end: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + blockDataDestroy(p); pTaskInfo->code = code; T_LONG_JMP(pTaskInfo->env, code); } @@ -1961,7 +2035,8 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan if (pInfo->tbnameSlotId != -1) { SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId); - char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; + QUERY_CHECK_NULL(pColumnInfoData, code, lino, _end, terrno); + char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(varTbName, name); code = colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows, true); @@ -2078,8 +2153,8 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca } } -int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - const char* pUser, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { +int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, + SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { QRY_OPTR_CHECK(pOptrInfo); int32_t code = TSDB_CODE_SUCCESS; @@ -2088,7 +2163,7 @@ int32_t createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNo SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - lino = __LINE__; + lino = __LINE__; goto _error; } @@ -2149,7 +2224,10 @@ _error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - destroyOperator(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -2184,7 +2262,7 @@ void destroySysScanOperator(void* param) { if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - if (pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) { + if (pInfo->pAPI != NULL && pInfo->pAPI->metaFn.closeTableMetaCursor != NULL) { pInfo->pAPI->metaFn.closeTableMetaCursor(pInfo->pCur); } @@ -2377,10 +2455,10 @@ static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k } int32_t optSysIntersection(SArray* in, SArray* out) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t lino = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; MergeIndex* mi = NULL; - int32_t sz = (int32_t)taosArrayGetSize(in); + int32_t sz = (int32_t)taosArrayGetSize(in); if (sz <= 0) { goto _end; } @@ -2617,6 +2695,7 @@ static int32_t doBlockInfoScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); + QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno); int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); @@ -2637,7 +2716,8 @@ static int32_t doBlockInfoScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes // make the valgrind happy that all memory buffer has been initialized already. if (slotId != 0) { SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0); - int64_t v = 0; + QUERY_CHECK_NULL(p1, code, lino, _end, terrno); + int64_t v = 0; colDataSetInt64(p1, 0, &v); } @@ -2663,7 +2743,9 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle); + if (pDistInfo->readHandle.api.tsdReader.tsdReaderClose != NULL) { + pDistInfo->readHandle.api.tsdReader.tsdReaderClose(pDistInfo->pHandle); + } tableListDestroy(pDistInfo->pTableListInfo); taosMemoryFreeClear(param); } @@ -2697,11 +2779,12 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC } int32_t createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, - STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { + STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, + SOperatorInfo** pOptrInfo) { QRY_OPTR_CHECK(pOptrInfo); - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2748,7 +2831,13 @@ int32_t createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanP return code; _error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo) { + pInfo->pTableListInfo = NULL; + destroyBlockDistScanOperatorInfo(pInfo); + } + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } return code; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index c237494f4c..3158c85987 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -47,6 +47,10 @@ static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, } SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx); + if (!pKey) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); + T_LONG_JMP(pFillInfo->pTaskInfo->env, terrno); + } int32_t code = doSetVal(pDstColInfo, rowIndex, pKey); if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); @@ -426,6 +430,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (pFillInfo->type == TSDB_FILL_PREV) { SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); + QUERY_CHECK_NULL(pKey, code, lino, _end, terrno); code = doSetVal(pDst, index, pKey); QUERY_CHECK_CODE(code, lino, _end); } else if (pFillInfo->type == TSDB_FILL_LINEAR) { @@ -440,6 +445,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t } else if (pFillInfo->type == TSDB_FILL_NEXT) { SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->next.pRowVal : pFillInfo->prev.pRowVal; SGroupKeys* pKey = taosArrayGet(p, i); + QUERY_CHECK_NULL(pKey, code, lino, _end, terrno); code = doSetVal(pDst, index, pKey); QUERY_CHECK_CODE(code, lino, _end); } else { @@ -582,12 +588,12 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { } for (int32_t i = 0; i < taosArrayGetSize(pFillInfo->prev.pRowVal); ++i) { SGroupKeys* pKey = taosArrayGet(pFillInfo->prev.pRowVal, i); - taosMemoryFree(pKey->pData); + if (pKey) taosMemoryFree(pKey->pData); } taosArrayDestroy(pFillInfo->prev.pRowVal); for (int32_t i = 0; i < taosArrayGetSize(pFillInfo->next.pRowVal); ++i) { SGroupKeys* pKey = taosArrayGet(pFillInfo->next.pRowVal, i); - taosMemoryFree(pKey->pData); + if (pKey) taosMemoryFree(pKey->pData); } taosArrayDestroy(pFillInfo->next.pRowVal); @@ -732,6 +738,8 @@ int64_t getFillInfoStart(struct SFillInfo* pFillInfo) { return pFillInfo->start; SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprInfo* pNotFillExpr, int32_t numOfNoFillExpr, const struct SNodeListNode* pValNode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; SFillColInfo* pFillCol = taosMemoryCalloc(numOfFillExpr + numOfNoFillExpr, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; @@ -749,6 +757,7 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn int32_t index = (i >= len) ? (len - 1) : i; SValueNode* pv = (SValueNode*)nodesListGetNode(pValNode->pNodeList, index); + QUERY_CHECK_NULL(pv, code, lino, _end, terrno); nodesValueNodeToVariant(pv, &pFillCol[i].fillVal); } } @@ -761,4 +770,14 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn } return pFillCol; + +_end: + for (int32_t i = 0; i < numOfFillExpr; ++i) { + taosVariantDestroy(&pFillCol[i].fillVal); + } + taosMemoryFree(pFillCol); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return NULL; } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index a4bf2dce72..65fcc4d4bc 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -1207,8 +1207,11 @@ _error: if (code != TSDB_CODE_SUCCESS) { qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); } - taosMemoryFree(pInfo); - taosMemoryFree(pOperator); + if (pInfo != NULL) destroyTimeSliceOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 7a5ab58331..ca6b89f7c5 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -75,9 +75,9 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup, true); - if (pResultRow == NULL) { + if (pResultRow == NULL || pTaskInfo->code != 0) { *pResult = NULL; - return TSDB_CODE_SUCCESS; + return pTaskInfo->code; } // set time window for current result @@ -614,6 +614,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + if (!pTsKey) { + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, terrno); + } + int64_t prevTs = *(int64_t*)pTsKey->pData; if (groupId == pBlock->info.id.groupId) { doTimeWindowInterpolation(pInfo->pPrevValues, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, w.ekey, @@ -661,7 +666,9 @@ static bool isCalculatedWin(SIntervalAggOperatorInfo* pInfo, const STimeWindow* * every tuple in every block. * And the boundedQueue keeps refreshing all records with smaller ts key. */ -static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STimeWindow* win, uint64_t groupId) { +static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STimeWindow* win, uint64_t groupId, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; if (!pOperatorInfo->limited // if no limit info, no filter will be applied || pOperatorInfo->binfo.inputTsOrder != pOperatorInfo->binfo.outputTsOrder // if input/output ts order mismatch, no filter @@ -673,6 +680,7 @@ static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STime if (pOperatorInfo->pBQ == NULL) { pOperatorInfo->pBQ = createBoundedQueue(pOperatorInfo->limit - 1, tsKeyCompFn, taosMemoryFree, pOperatorInfo); + QUERY_CHECK_NULL(pOperatorInfo->pBQ, code, lino, _end, terrno); } bool shouldFilter = false; @@ -689,12 +697,21 @@ static bool filterWindowWithLimit(SIntervalAggOperatorInfo* pOperatorInfo, STime // cur win not been filtered out and not been pushed into BQ yet, push it into BQ PriorityQueueNode node = {.data = taosMemoryMalloc(sizeof(TSKEY))}; + QUERY_CHECK_NULL(node.data, code, lino, _end, terrno); + *((TSKEY*)node.data) = win->skey; if (NULL == taosBQPush(pOperatorInfo->pBQ, &node)) { taosMemoryFree(node.data); return true; } + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + pTaskInfo->code = code; + T_LONG_JMP(pTaskInfo->env, code); + } return false; } @@ -726,7 +743,7 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->binfo.inputTsOrder); - if (filterWindowWithLimit(pInfo, &win, tableGroupId)) return false; + if (filterWindowWithLimit(pInfo, &win, tableGroupId, pTaskInfo)) return false; int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); @@ -765,7 +782,7 @@ static bool hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t prevEndPos = forwardRows - 1 + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->binfo.inputTsOrder); - if (startPos < 0 || filterWindowWithLimit(pInfo, &nextWin, tableGroupId)) { + if (startPos < 0 || filterWindowWithLimit(pInfo, &nextWin, tableGroupId, pTaskInfo)) { break; } // null data, failed to allocate more memory buffer @@ -846,6 +863,11 @@ int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo if (pBlock->pDataBlock != NULL && pBlock->info.dataLoad) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + if (!pColDataInfo) { + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, terrno); + } + tsCols = (int64_t*)pColDataInfo->pData; ASSERT(tsCols[0] != 0); @@ -924,6 +946,10 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SExprSupp* pSup = &pOperator->exprSupp; SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId); + if (!pStateColInfoData) { + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, terrno); + } int64_t gid = pBlock->info.id.groupId; bool masterScan = true; @@ -931,6 +957,10 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t bytes = pStateColInfoData->info.bytes; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + if (!pColInfoData) { + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, terrno); + } TSKEY* tsList = (TSKEY*)pColInfoData->pData; SWindowRowsSup* pRowSup = &pInfo->winSup; @@ -1384,7 +1414,10 @@ _error: if (pInfo != NULL) { destroyIntervalOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -1395,6 +1428,10 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator SExprSupp* pSup = &pOperator->exprSupp; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + if (!pColInfoData) { + pTaskInfo->code = terrno; + T_LONG_JMP(pTaskInfo->env, terrno); + } bool masterScan = true; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; @@ -1656,7 +1693,10 @@ _error: destroyStateWindowOperatorInfo(pInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -1706,9 +1746,7 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); pInfo->twAggSup.waterMark = pSessionNode->window.watermark; pInfo->twAggSup.calTrigger = pSessionNode->window.triggerType; @@ -1732,15 +1770,11 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh QUERY_CHECK_CODE(code, lino, _error); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); } code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -1748,16 +1782,17 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + QUERY_CHECK_CODE(code, lino, _error); *pOptrInfo = pOperator; return code; _error: - destroySWindowOperatorInfo(pInfo); - taosMemoryFreeClear(pOperator); + if (pInfo != NULL) destroySWindowOperatorInfo(pInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -2068,8 +2103,11 @@ int32_t createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMerge return code; _error: - destroyMAIOperatorInfo(miaInfo); - taosMemoryFreeClear(pOperator); + if (miaInfo != NULL) destroyMAIOperatorInfo(miaInfo); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } @@ -2401,7 +2439,10 @@ _error: destroyMergeIntervalOperatorInfo(pMergeIntervalInfo); } - taosMemoryFreeClear(pOperator); + if (pOperator != NULL) { + pOperator->info = NULL; + destroyOperator(pOperator); + } pTaskInfo->code = code; return code; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 896b4db7cd..cedc23ed2d 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -152,8 +152,10 @@ static void destoryAllocatedTuple(void* t) { taosMemoryFree(t); } * @param colIndex the columnIndex, for setting null bitmap * @return the next offset to add field * */ -static inline size_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, uint32_t colIdx, void* data, - size_t length, bool isNull, uint32_t tupleLen) { +static inline int32_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, uint32_t colIdx, void* data, + size_t length, bool isNull, uint32_t tupleLen, uint32_t* pOffset) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t lino = 0; tupleSetOffset(*t, colIdx, offset); if (isNull) { @@ -161,16 +163,20 @@ static inline size_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, u } else { if (offset + length > tupleLen + tupleGetDataStartOffset(colNum)) { void* px = taosMemoryRealloc(*t, offset + length); - if (px == NULL) { - return terrno; - } + QUERY_CHECK_NULL(px, code, lino, _end, terrno); *t = px; } tupleSetData(*t, offset, data, length); } - return offset + length; + (*pOffset) = offset + length; + +_end: + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; } static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) { @@ -182,8 +188,8 @@ static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) { } int32_t tsortGetSortedDataBlock(const SSortHandle* pSortHandle, SSDataBlock** pBlock) { + *pBlock = NULL; if (pSortHandle->pDataBlock == NULL) { - *pBlock = NULL; return TSDB_CODE_SUCCESS; } return createOneDataBlock(pSortHandle->pDataBlock, false, pBlock); @@ -200,6 +206,7 @@ typedef struct ReferencedTuple { } ReferencedTuple; static int32_t createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t tupleLen, size_t rowIdx, TupleDesc** pDesc) { + int32_t code = TSDB_CODE_SUCCESS; TupleDesc* t = taosMemoryCalloc(1, sizeof(TupleDesc)); if (t == NULL) { return terrno; @@ -216,15 +223,20 @@ static int32_t createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t for (size_t colIdx = 0; colIdx < colNum; ++colIdx) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx); if (pCol == NULL) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); return terrno; } if (colDataIsNull_s(pCol, rowIdx)) { - offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen); + code = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen, &offset); } else { colLen = colDataGetRowLength(pCol, rowIdx); - offset = - tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, tupleLen); + code = + tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, tupleLen, &offset); + } + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); + return code; } } @@ -232,7 +244,7 @@ static int32_t createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t t->data = pTuple; *pDesc = t; - return 0; + return code; } int32_t tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNum, void** pResult) { @@ -259,7 +271,7 @@ int32_t tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNu void destroyTuple(void* t) { TupleDesc* pDesc = t; - if (pDesc->type == AllocatedTupleType) { + if (pDesc != NULL && pDesc->type == AllocatedTupleType) { destoryAllocatedTuple(pDesc->data); taosMemoryFree(pDesc); } @@ -285,6 +297,10 @@ int32_t tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, pSortHandle->pageSize = pageSize; pSortHandle->numOfPages = numOfPages; pSortHandle->pSortInfo = taosArrayDup(pSortInfo, NULL); + if (pSortHandle->pSortInfo == NULL) { + return terrno; + } + pSortHandle->loops = 0; pSortHandle->pqMaxTupleLength = pqMaxTupleLength; @@ -469,10 +485,10 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortExternalBuf", tsTempDir); - dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; } + dBufSetPrintInfo(pHandle->pBuf); } SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t)); @@ -562,10 +578,10 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "sortComparInit", tsTempDir); - dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { - terrno = code; return code; + } else { + dBufSetPrintInfo(pHandle->pBuf); } } @@ -1111,9 +1127,10 @@ static int32_t createPageBuf(SSortHandle* pHandle) { int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, "tableBlocksBuf", tsTempDir); - dBufSetPrintInfo(pHandle->pBuf); if (code != TSDB_CODE_SUCCESS) { return code; + } else { + dBufSetPrintInfo(pHandle->pBuf); } } return 0; @@ -1681,6 +1698,7 @@ static int32_t initRowIdSort(SSortHandle* pHandle) { biTs.compFn = getKeyComparFunc(TSDB_DATA_TYPE_TIMESTAMP, biTs.order); void* p = taosArrayPush(pOrderInfoList, &biTs); if (p == NULL) { + taosArrayDestroy(pOrderInfoList); return terrno; } @@ -1693,6 +1711,7 @@ static int32_t initRowIdSort(SSortHandle* pHandle) { void* px = taosArrayPush(pOrderInfoList, &biPk); if (px == NULL) { + taosArrayDestroy(pOrderInfoList); return terrno; } } @@ -1707,6 +1726,9 @@ int32_t tsortSetSortByRowId(SSortHandle* pHandle, int32_t extRowsMemSize) { pHandle->extRowBytes = blockDataGetRowSize(pHandle->pDataBlock) + taosArrayGetSize(pHandle->pDataBlock->pDataBlock) + sizeof(int32_t); pHandle->extRowsMemSize = extRowsMemSize; pHandle->aExtRowsOrders = taosArrayDup(pHandle->pSortInfo, NULL); + if (pHandle->aExtRowsOrders == NULL) { + return terrno; + } int32_t code = initRowIdSort(pHandle); if (code) { @@ -2452,7 +2474,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) { return code; } -static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) { +static int32_t tsortOpenForBufMergeSort(SSortHandle* pHandle) { int32_t code = createInitialSources(pHandle); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2478,7 +2500,8 @@ static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) { return code; } - return tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + return code; } void tsortClose(SSortHandle* pHandle) { @@ -2808,19 +2831,24 @@ static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle } int32_t tsortOpen(SSortHandle* pHandle) { + int32_t code = 0; if (pHandle->opened) { - return 0; + return code; } - if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { - return TSDB_CODE_INVALID_PARA; + if (pHandle == NULL || pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { + code = TSDB_CODE_INVALID_PARA; + return code; } pHandle->opened = true; - if (tsortIsPQSortApplicable(pHandle)) - return tsortOpenForPQSort(pHandle); - else - return tsortOpenForBufMergeSort(pHandle); + if (tsortIsPQSortApplicable(pHandle)) { + code = tsortOpenForPQSort(pHandle); + } else { + code = tsortOpenForBufMergeSort(pHandle); + } + + return code; } int32_t tsortNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 40e0407a54..4eefd150f3 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -305,7 +305,7 @@ int32_t tMemBucketCreate(int32_t nElemSize, int16_t dataType, double minval, dou return TSDB_CODE_NO_DISKSPACE; } - int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * 1024, "1", tsTempDir); + int32_t ret = createDiskbasedBuf(&(*pBucket)->pBuffer, (*pBucket)->bufPageSize, (*pBucket)->bufPageSize * DEFAULT_NUM_OF_SLOT * 4, "1", tsTempDir); if (ret != 0) { tMemBucketDestroy(*pBucket); return ret; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ad9e5ce7d4..9a751db801 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -162,7 +162,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn); } else { fnError("[UDFD]Failed to allocate memory for TAOS_FQDN"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } } @@ -837,10 +837,13 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfBlock->numOfRows = block->info.rows; udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock); udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn *)); + if((udfBlock->udfCols) == NULL) { + return terrno; + } for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); if(udfBlock->udfCols[i] == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SColumnInfoData *col = (SColumnInfoData *)taosArrayGet(block->pDataBlock, i); SUdfColumn *udfCol = udfBlock->udfCols[i]; @@ -854,13 +857,13 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows; udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen); if(udfCol->colData.varLenCol.varOffsets == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); udfCol->colData.varLenCol.payloadLen = colDataGetLength(col, udfBlock->numOfRows); udfCol->colData.varLenCol.payload = taosMemoryMalloc(udfCol->colData.varLenCol.payloadLen); if(udfCol->colData.varLenCol.payload == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (col->reassigned) { for (int32_t row = 0; row < udfCol->colData.numOfRows; ++row) { @@ -882,7 +885,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo int32_t bitmapLen = udfCol->colData.fixLenCol.nullBitmapLen; udfCol->colData.fixLenCol.nullBitmap = taosMemoryMalloc(udfCol->colData.fixLenCol.nullBitmapLen); if(udfCol->colData.fixLenCol.nullBitmap == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } char *bitmap = udfCol->colData.fixLenCol.nullBitmap; memcpy(bitmap, col->nullbitmap, bitmapLen); @@ -985,7 +988,7 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { output->columnData = taosMemoryMalloc(sizeof(SColumnInfoData)); if(output->columnData == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } memcpy(output->columnData, taosArrayGet(input->pDataBlock, 0), sizeof(SColumnInfoData)); output->colAlloced = true; @@ -1724,7 +1727,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { if(conn == NULL) { fnError("udfc event loop start connect task malloc conn failed."); taosMemoryFree(pipe); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } conn->pipe = pipe; conn->readBuf.len = 0; @@ -1954,7 +1957,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); if(uvTask == NULL) { fnError("udfc client task: %p failed to allocate memory for uvTask", task); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } fnDebug("udfc client task: %p created uvTask: %p. pipe: %p", task, uvTask, task->session->udfUvPipe); @@ -1986,13 +1989,13 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); if(task == NULL) { fnError("doSetupUdf, failed to allocate memory for task"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); if(task->session == NULL) { fnError("doSetupUdf, failed to allocate memory for session"); taosMemoryFree(task); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } task->session->udfc = &gUdfcProxy; task->type = UDF_TASK_SETUP; @@ -2037,7 +2040,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); if(task == NULL) { fnError("udfc call udf. failed to allocate memory for task"); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } task->session = (SUdfcUvSession *)handle; task->type = UDF_TASK_CALL; @@ -2169,7 +2172,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if(task == NULL) { fnError("doTeardownUdf, failed to allocate memory for task"); taosMemoryFree(session); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } task->session = session; task->type = UDF_TASK_TEARDOWN; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 7339f115a3..2d8a926c72 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -409,6 +409,10 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) { int16_t lenPythonPath = strlen(tsUdfdLdLibPath) + strlen(global.udfDataDir) + 1 + 1; // global.udfDataDir:tsUdfdLdLibPath char *pythonPath = taosMemoryMalloc(lenPythonPath); + if(pythonPath == NULL) { + uv_dlclose(&plugin->lib); + return terrno; + } #ifdef WINDOWS snprintf(pythonPath, lenPythonPath, "%s;%s", global.udfDataDir, tsUdfdLdLibPath); #else @@ -705,6 +709,10 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { uv_mutex_unlock(&udf->lock); } SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); + if(handle == NULL) { + fnError("udfdProcessSetupRequest: malloc failed."); + code = terrno; + } handle->udf = udf; _send: @@ -775,7 +783,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { if (outBuf.buf != NULL) { code = udf->scriptPlugin->udfAggStartFunc(&outBuf, udf->scriptUdfCtx); } else { - code = TSDB_CODE_OUT_OF_MEMORY; + code = terrno; } subRsp->resultBuf = outBuf; break; @@ -784,9 +792,13 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfDataBlock input = {0}; if (convertDataBlockToUdfDataBlock(&call->block, &input) == TSDB_CODE_SUCCESS) { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx); - freeUdfInterBuf(&call->interBuf); - subRsp->resultBuf = outBuf; + if (outBuf.buf != NULL) { + code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx); + freeUdfInterBuf(&call->interBuf); + subRsp->resultBuf = outBuf; + } else { + code = terrno; + } } freeUdfDataDataBlock(&input); @@ -794,18 +806,27 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } case TSDB_UDF_CALL_AGG_MERGE: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->scriptPlugin->udfAggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf, udf->scriptUdfCtx); - freeUdfInterBuf(&call->interBuf); - freeUdfInterBuf(&call->interBuf2); - subRsp->resultBuf = outBuf; + if (outBuf.buf != NULL) { + code = udf->scriptPlugin->udfAggMergeFunc(&call->interBuf, &call->interBuf2, &outBuf, udf->scriptUdfCtx); + freeUdfInterBuf(&call->interBuf); + freeUdfInterBuf(&call->interBuf2); + subRsp->resultBuf = outBuf; + } else { + code = terrno; + } break; } case TSDB_UDF_CALL_AGG_FIN: { SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0}; - code = udf->scriptPlugin->udfAggFinishFunc(&call->interBuf, &outBuf, udf->scriptUdfCtx); - freeUdfInterBuf(&call->interBuf); - subRsp->resultBuf = outBuf; + if (outBuf.buf != NULL) { + code = udf->scriptPlugin->udfAggFinishFunc(&call->interBuf, &outBuf, udf->scriptUdfCtx); + freeUdfInterBuf(&call->interBuf); + subRsp->resultBuf = outBuf; + } else { + code = terrno; + } + break; } default: @@ -820,19 +841,24 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { int32_t len = encodeUdfResponse(NULL, rsp); if(len < 0) { fnError("udfdProcessCallRequest: encode udf response failed. len %d", len); - return; + goto _exit; } rsp->msgLen = len; void *bufBegin = taosMemoryMalloc(len); + if (bufBegin == NULL) { + fnError("udfdProcessCallRequest: malloc failed. len %d", len); + goto _exit; + } void *buf = bufBegin; if(encodeUdfResponse(&buf, rsp) < 0) { fnError("udfdProcessCallRequest: encode udf response failed. len %d", len); taosMemoryFree(bufBegin); - return; + goto _exit; } uvUdf->output = uv_buf_init(bufBegin, len); +_exit: switch (call->callType) { case TSDB_UDF_CALL_SCALA_PROC: { blockDataFreeRes(&call->block); @@ -906,6 +932,10 @@ _send: } rsp->msgLen = len; void *bufBegin = taosMemoryMalloc(len); + if(bufBegin == NULL) { + fnError("udfdProcessTeardownRequest: malloc failed. len %d", len); + return; + } void *buf = bufBegin; if (encodeUdfResponse(&buf, rsp) < 0) { fnError("udfdProcessTeardownRequest: encode udf response failed. len %d", len); @@ -1173,7 +1203,7 @@ int32_t udfdOpenClientRpc() { global.clientRpc = rpcOpen(&rpcInit); if (global.clientRpc == NULL) { fnError("failed to init dnode rpc client"); - return -1; + return terrno; } return 0; } @@ -1210,6 +1240,11 @@ void udfdSendResponse(uv_work_t *work, int status) { if (udfWork->conn != NULL) { uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + if(write_req == NULL) { + fnError("udfd send response error, malloc failed"); + taosMemoryFree(work); + return; + } write_req->data = udfWork; int32_t code = uv_write(write_req, udfWork->conn->client, &udfWork->output, 1, udfdOnWrite); if (code != 0) { @@ -1269,7 +1304,16 @@ void udfdHandleRequest(SUdfdUvConn *conn) { int32_t inputLen = conn->inputLen; uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); + if(work == NULL) { + fnError("udfd malloc work failed"); + return; + } SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); + if(udfWork == NULL) { + fnError("udfd malloc udf work failed"); + taosMemoryFree(work); + return; + } udfWork->conn = conn; udfWork->pWorkNext = conn->pWorkList; conn->pWorkList = udfWork; @@ -1334,6 +1378,10 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { int32_t code = 0; uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t)); + if(client == NULL) { + fnError("udfd pipe malloc failed"); + return; + } code = uv_pipe_init(global.loop, client, 0); if (code) { fnError("udfd pipe init error %s", uv_strerror(code)); @@ -1342,6 +1390,10 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { } if (uv_accept(server, (uv_stream_t *)client) == 0) { SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + if(ctx == NULL) { + fnError("udfd conn malloc failed"); + goto _exit; + } ctx->pWorkList = NULL; ctx->client = (uv_stream_t *)client; ctx->inputBuf = 0; @@ -1356,9 +1408,11 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { taosMemoryFree(ctx); taosMemoryFree(client); } - } else { - uv_close((uv_handle_t *)client, NULL); + return; } +_exit: + uv_close((uv_handle_t *)client, NULL); + taosMemoryFree(client); } void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { @@ -1411,6 +1465,10 @@ static int32_t udfdInitLog() { void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { buf->base = taosMemoryMalloc(suggested_size); + if (buf->base == NULL) { + fnError("udfd ctrl pipe alloc buffer failed"); + return; + } buf->len = suggested_size; } @@ -1477,13 +1535,13 @@ static int32_t udfdGlobalDataInit() { uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t)); if (loop == NULL) { fnError("udfd init uv loop failed, mem overflow"); - return -1; + return terrno; } global.loop = loop; if (uv_mutex_init(&global.scriptPluginsMutex) != 0) { fnError("udfd init script plugins mutex failed"); - return -1; + return TSDB_CODE_UDF_UV_EXEC_FAILURE; } global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); @@ -1494,7 +1552,7 @@ static int32_t udfdGlobalDataInit() { if (uv_mutex_init(&global.udfsMutex) != 0) { fnError("udfd init udfs mutex failed"); - return -2; + return TSDB_CODE_UDF_UV_EXEC_FAILURE; } return 0; diff --git a/source/libs/geometry/src/geomFunc.c b/source/libs/geometry/src/geomFunc.c index 194590c06c..4426427bf5 100644 --- a/source/libs/geometry/src/geomFunc.c +++ b/source/libs/geometry/src/geomFunc.c @@ -156,6 +156,10 @@ _exit: int32_t executeGeomFromTextFunc(SColumnInfoData *pInputData, int32_t i, SColumnInfoData *pOutputData) { int32_t code = TSDB_CODE_FAILED; + if (!IS_VAR_DATA_TYPE((pInputData)->info.type)) { + return TSDB_CODE_FUNC_FUNTION_PARA_VALUE; + } + char *input = colDataGetData(pInputData, i); unsigned char *output = NULL; diff --git a/source/libs/geometry/src/geosWrapper.c b/source/libs/geometry/src/geosWrapper.c index 6ca8a39bb5..dde34edc91 100644 --- a/source/libs/geometry/src/geosWrapper.c +++ b/source/libs/geometry/src/geosWrapper.c @@ -147,7 +147,17 @@ static int32_t initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( " "*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$"); - code = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace); + pcre2_code *pRegex = NULL; + pcre2_match_data *pMatchData = NULL; + code = doRegComp(&pRegex, &pMatchData, wktPatternWithSpace); + if (code < 0) { + taosMemoryFree(wktPatternWithSpace); + return TSDB_CODE_OUT_OF_MEMORY; + } + + *ppRegex = pRegex; + *ppMatchData = pMatchData; + taosMemoryFree(wktPatternWithSpace); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index bce8cda125..d47a183121 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -225,6 +225,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra .code = 0 }; TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); + int code = rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); if (code) { destroySendMsgInfo(pInfo); @@ -235,6 +236,9 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, SMsgSendInfo* pInfo) { return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL); } +int32_t asyncFreeConnById(void* pTransporter, int64_t pid) { + return rpcFreeConnById(pTransporter, pid); +} char* jobTaskStatusStr(int32_t status) { switch (status) { @@ -448,13 +452,13 @@ void parseTagDatatoJson(void* p, char** jsonStr) { if (value == NULL) { goto end; } - if(!cJSON_AddItemToObject(json, tagJsonKey, value)){ + if (!cJSON_AddItemToObject(json, tagJsonKey, value)) { goto end; } } else if (type == TSDB_DATA_TYPE_NCHAR) { cJSON* value = NULL; if (pTagVal->nData > 0) { - char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); + char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); if (tagJsonValue == NULL) { goto end; } @@ -479,7 +483,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) { goto end; } - if(!cJSON_AddItemToObject(json, tagJsonKey, value)){ + if (!cJSON_AddItemToObject(json, tagJsonKey, value)) { goto end; } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -488,7 +492,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) { if (value == NULL) { goto end; } - if(!cJSON_AddItemToObject(json, tagJsonKey, value)){ + if (!cJSON_AddItemToObject(json, tagJsonKey, value)) { goto end; } } else if (type == TSDB_DATA_TYPE_BOOL) { @@ -497,7 +501,7 @@ void parseTagDatatoJson(void* p, char** jsonStr) { if (value == NULL) { goto end; } - if(!cJSON_AddItemToObject(json, tagJsonKey, value)){ + if (!cJSON_AddItemToObject(json, tagJsonKey, value)) { goto end; } } else { @@ -581,6 +585,8 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, sizeof(*pSrc)); + (*pDst)->vgArray = NULL; + if (pSrc->vgHash) { (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index e8deed1df9..207bd91bd9 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -34,7 +34,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; @@ -509,7 +509,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } SSchemaExt *pSchemaExt = (SSchemaExt *)((char *)pTableMeta + metaSize); @@ -764,7 +764,7 @@ int32_t queryProcessGetTbCfgRsp(void *output, char *msg, int32_t msgSize) { STableCfgRsp *out = taosMemoryCalloc(1, sizeof(STableCfgRsp)); if(out == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tDeserializeSTableCfgRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableCfgRsp failed, msgSize:%d", msgSize); @@ -785,7 +785,7 @@ int32_t queryProcessGetViewMetaRsp(void *output, char *msg, int32_t msgSize) { SViewMetaRsp *out = taosMemoryCalloc(1, sizeof(SViewMetaRsp)); if (out == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + return terrno; } if (tDeserializeSViewMetaRsp(msg, msgSize, out) != 0) { qError("tDeserializeSViewMetaRsp failed, msgSize:%d", msgSize); diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index ab04f06b02..4d45fb344c 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -464,7 +464,7 @@ struct SFilterInfo { (colInfo).type = RANGE_TYPE_UNIT; \ (colInfo).dataType = FILTER_UNIT_DATA_TYPE(u); \ if (taosArrayPush((SArray *)((colInfo).info), &u) == NULL) { \ - FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); \ + FLT_ERR_RET(terrno); \ } \ } while (0) #define FILTER_PUSH_VAR_HASH(colInfo, ha) \ @@ -481,6 +481,9 @@ struct SFilterInfo { #define FILTER_COPY_IDX(dst, src, n) \ do { \ *(dst) = taosMemoryMalloc(sizeof(uint32_t) * n); \ + if (NULL == *(dst)) { \ + FLT_ERR_JRET(terrno); \ + } \ (void)memcpy(*(dst), src, sizeof(uint32_t) * n); \ } while (0) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5226668f82..e5d0fe594a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -2343,7 +2343,7 @@ _return: (void)filterFreeRangeCtx(ctx); // No need to handle the return value. - return TSDB_CODE_SUCCESS; + return code; } int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gResNum) { @@ -2671,7 +2671,7 @@ _return: (void)filterFreeRangeCtx(ctx); // No need to handle the return value. - return TSDB_CODE_SUCCESS; + return code; } int32_t filterMergeGroups(SFilterInfo *info, SFilterGroupCtx **gRes, int32_t *gResNum) { @@ -2758,7 +2758,7 @@ _return: FILTER_SET_FLAG(info->status, FI_STATUS_ALL); - return TSDB_CODE_SUCCESS; + return code; } int32_t filterConvertGroupFromArray(SFilterInfo *info, SArray *group) { @@ -2958,7 +2958,7 @@ _return: taosMemoryFreeClear(idxNum); taosMemoryFreeClear(idxs); - return TSDB_CODE_SUCCESS; + return code; } int32_t filterPostProcessRange(SFilterInfo *info) { @@ -3601,12 +3601,12 @@ int32_t filterPreprocess(SFilterInfo *info) { if (FILTER_GET_FLAG(info->status, FI_STATUS_ALL)) { fltInfo("Final - FilterInfo: [ALL]"); - goto _return; + goto _return1; } if (FILTER_GET_FLAG(info->status, FI_STATUS_EMPTY)) { fltInfo("Final - FilterInfo: [EMPTY]"); - goto _return; + goto _return1; } FLT_ERR_JRET(filterGenerateColRange(info, gRes, gResNum)); @@ -3619,10 +3619,10 @@ int32_t filterPreprocess(SFilterInfo *info) { FLT_ERR_JRET(filterGenerateComInfo(info)); +_return1: + FLT_ERR_JRET(filterSetExecFunc(info)); + _return: - - FLT_ERR_RET(filterSetExecFunc(info)); - for (int32_t i = 0; i < gResNum; ++i) { filterFreeGroupCtx(gRes[i]); } @@ -3660,15 +3660,25 @@ int32_t fltInitFromNode(SNode *tree, SFilterInfo *info, uint32_t options) { FLT_ERR_JRET(terrno); } - FLT_ERR_JRET(filterInitUnitsFields(info)); + code = filterInitUnitsFields(info); + if(TSDB_CODE_SUCCESS != code) { + taosArrayDestroy(group); + goto _return; + } SFltBuildGroupCtx tctx = {.info = info, .group = group}; nodesWalkExpr(tree, fltTreeToGroup, (void *)&tctx); - FLT_ERR_JRET(tctx.code); - - FLT_ERR_JRET(filterConvertGroupFromArray(info, group)); + if (TSDB_CODE_SUCCESS != tctx.code) { + taosArrayDestroy(group); + code = tctx.code; + goto _return; + } + code = filterConvertGroupFromArray(info, group); + if (TSDB_CODE_SUCCESS != code) { + taosArrayDestroy(group); + goto _return; + } taosArrayDestroy(group); - FLT_ERR_JRET(fltInitValFieldData(info)); if (!FILTER_GET_FLAG(info->options, FLT_OPTION_NO_REWRITE)) { @@ -4993,7 +5003,7 @@ int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) } _return: taosArrayDestroy(sclOpList); - return TSDB_CODE_SUCCESS; + return code; } int32_t fltGetDataFromColId(void *param, int32_t id, void **data) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d08f358ce0..9428f051aa 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -904,9 +904,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp terrno = TSDB_CODE_SUCCESS; SCL_ERR_JRET(OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC)); - SCL_ERR_JRET(terrno); -_return: +_return: sclFreeParamList(params, paramNum); SCL_RET(code); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 23cc7324f0..5db7aebeee 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -401,8 +401,8 @@ static int32_t concatCopyHelper(const char *input, char *output, bool hasNchar, taosMemoryFree(newBuf); return TSDB_CODE_SCALAR_CONVERT_ERROR; } - (void)memcpy(varDataVal(output) + *dataLen, newBuf, varDataLen(input) * TSDB_NCHAR_SIZE); - *dataLen += varDataLen(input) * TSDB_NCHAR_SIZE; + (void)memcpy(varDataVal(output) + *dataLen, newBuf, len); + *dataLen += len; taosMemoryFree(newBuf); } else { (void)memcpy(varDataVal(output) + *dataLen, varDataVal(input), varDataLen(input)); @@ -2913,7 +2913,7 @@ int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP _return: taosMemoryFree(bins); - return TSDB_CODE_SUCCESS; + return code; } int32_t selectScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 08a8f684f5..9215254f9c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -17,11 +17,11 @@ #include "command.h" #include "query.h" #include "schInt.h" +#include "tglobal.h" +#include "tmisce.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" -#include "tglobal.h" -#include "tmisce.h" // clang-format off int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { @@ -975,11 +975,13 @@ int32_t schAsyncSendMsg(SSchJob *pJob, SSchTask *pTask, SSchTrans *trans, SQuery SCH_ERR_JRET(schUpdateSendTargetInfo(pMsgSendInfo, addr, pTask)); if (isHb && persistHandle && trans->pHandle == 0) { - trans->pHandle = rpcAllocHandle(); - if (NULL == trans->pHandle) { - SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", terrno); - SCH_ERR_JRET(terrno); + int64_t refId = 0; + code = rpcAllocHandle(&refId); + if (code != 0) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", code); + SCH_ERR_JRET(code); } + trans->pHandle = (void *)refId; } if (pJob && pTask) { @@ -1200,7 +1202,14 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } persistHandle = true; - SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); + int64_t refId = 0; + code = rpcAllocHandle(&refId); + if (code != 0) { + SCH_TASK_ELOG("rpcAllocHandle failed, code:%x", code); + SCH_ERR_JRET(code); + } + + SCH_SET_TASK_HANDLE(pTask, (void *)refId); break; } case TDMT_SCH_FETCH: diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 832bcb5015..d3db0347e2 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -819,10 +819,21 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { uint32_t dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20; SBackendWrapper* pHandle = taosMemoryCalloc(1, sizeof(SBackendWrapper)); + if (pHandle == NULL) { + goto _EXIT; + } + pHandle->list = tdListNew(sizeof(SCfComparator)); + if (pHandle->list == NULL) { + goto _EXIT; + } + (void)taosThreadMutexInit(&pHandle->mutex, NULL); (void)taosThreadMutexInit(&pHandle->cfMutex, NULL); pHandle->cfInst = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pHandle->cfInst == NULL) { + goto _EXIT; + } rocksdb_env_t* env = rocksdb_create_default_env(); // rocksdb_envoptions_create(); @@ -1133,6 +1144,8 @@ int32_t chkpMayDelObsolete(void* arg, int64_t chkpId, char* path) { int64_t id = *(int64_t*)taosArrayGet(chkpDel, i); char tbuf[256] = {0}; sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); + + stInfo("backend remove obsolete checkpoint: %s", tbuf); if (taosIsDir(tbuf)) { taosRemoveDir(tbuf); } @@ -2650,6 +2663,7 @@ void taskDbDestroy(void* pDb, bool flush) { if (wrapper->removeAllFiles) { char* err = NULL; + stInfo("drop task remove backend dat:%s", wrapper->path); taosRemoveDir(wrapper->path); } taosMemoryFree(wrapper->path); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 9a2323582c..2de86b8794 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -74,13 +74,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ } if (pInfo->stage != stage) { - streamMutexLock(&pTask->lock); - ETaskStatus status = streamTaskGetStatus(pTask).state; - if (status == TASK_STATUS__CK) { - streamTaskSetFailedCheckpointId(pTask); - } - streamMutexUnlock(&pTask->lock); - return TASK_UPSTREAM_NEW_STAGE; } else if (pTask->status.downstreamReady != 1) { stDebug("s-task:%s vgId:%d leader:%d, downstream not ready", id, vgId, (pTask->pMeta->role == NODE_ROLE_LEADER)); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index acac5dfc9e..9da7a5d9c8 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -453,8 +453,9 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo == NULL) { - streamMutexUnlock(&pInfo->lock); - return TSDB_CODE_INVALID_PARA; + stError("s-task:%s invalid index during iterate the checkpoint-ready msg list, index:%d, ignore and continue", + pTask->id.idStr, i); + continue; } if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { @@ -468,8 +469,9 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo == NULL) { - streamMutexUnlock(&pInfo->lock); - return TSDB_CODE_INVALID_PARA; + stError("s-task:%s invalid index during iterate the checkpoint-ready msg list, index:%d, ignore and continue", + pTask->id.idStr, i); + continue; } if (pReadyInfo->sendCompleted == 1) { @@ -601,9 +603,15 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV } void streamTaskSetFailedCheckpointId(SStreamTask* pTask) { - pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId; - stDebug("s-task:%s mark the checkpointId:%" PRId64 " (transId:%d) failed", pTask->id.idStr, - pTask->chkInfo.pActiveInfo->activeId, pTask->chkInfo.pActiveInfo->transId); + struct SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + + if (pInfo->activeId <= 0) { + stWarn("s-task:%s checkpoint-info is cleared now, not set the failed checkpoint info", pTask->id.idStr); + } else { + pInfo->failedId = pInfo->activeId; + stDebug("s-task:%s mark the checkpointId:%" PRId64 " (transId:%d) failed", pTask->id.idStr, pInfo->activeId, + pInfo->transId); + } } static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* list) { @@ -960,6 +968,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) const char* id = pTask->id.idStr; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SStreamTaskState pStatus = streamTaskGetStatus(pTask); + bool alreadySend = false; if (pStatus.state != TASK_STATUS__CK) { return false; @@ -971,11 +980,12 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) return false; } - for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + int32_t num = taosArrayGetSize(pInfo->pDispatchTriggerList); + for (int32_t i = 0; i < num; ++i) { STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i); if (pSendInfo == NULL) { - streamMutexUnlock(&pInfo->lock); - return TSDB_CODE_INVALID_PARA; + stError("s-task:%s invalid index in dispatch-trigger list, index:%d, size:%d, ignore and continue", id, i, num); + continue; } if (pSendInfo->nodeId != downstreamNodeId) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 87293c59ec..fbcc338f09 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1140,6 +1140,20 @@ void streamMetaNotifyClose(SStreamMeta* pMeta) { taosMsleep(100); } + streamMetaRLock(pMeta); + + SArray* pTaskList = NULL; + int32_t code = streamMetaSendMsgBeforeCloseTasks(pMeta, &pTaskList); + if (code != TSDB_CODE_SUCCESS) { +// return code; + } + + streamMetaRUnLock(pMeta); + + if (pTaskList != NULL) { + taosArrayDestroy(pTaskList); + } + int64_t el = taosGetTimestampMs() - st; stDebug("vgId:%d all stream tasks are not in timer, continue close, elapsed time:%" PRId64 " ms", pMeta->vgId, el); } @@ -1277,10 +1291,13 @@ static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64 if (pMeta->closeFlag) { streamMetaWUnLock(pMeta); stError("vgId:%d vnode is closed, not start check task(s) downstream status", pMeta->vgId); - return -1; + return TSDB_CODE_FAILED; } *pList = taosArrayDup(pMeta->pTaskList, NULL); + if (*pList == NULL) { + return terrno; + } taosHashClear(pMeta->startInfo.pReadyTaskSet); taosHashClear(pMeta->startInfo.pFailedTaskSet); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 5e538c1e42..752101afbd 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -235,7 +235,7 @@ EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueIte void* newRet = NULL; int32_t code = streamQueueMergeQueueItem(*pInput, qItem, (SStreamQueueItem**)&newRet); if (newRet == NULL) { - if (code) { + if (code != -1) { stError("s-task:%s failed to merge blocks from inputQ, numOfBlocks:%d, code:%s", id, *numOfBlocks, tstrerror(code)); } diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index a83a0e4cc8..6506d449a6 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -48,7 +48,7 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, - terrstr(terrno)); + terrstr()); return terrno; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 59ce9e8d42..21362ca806 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -321,7 +321,7 @@ void streamFreeTaskState(SStreamTask* pTask, int8_t remove) { stDebug("s-task:0x%x start to free task state", pTask->id.taskId); streamStateClose(pTask->pState, remove); - taskDbSetClearFileFlag(pTask->pBackend); + if (remove)taskDbSetClearFileFlag(pTask->pBackend); taskDbRemoveRef(pTask->pBackend); pTask->pBackend = NULL; pTask->pState = NULL; diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index 11894f7853..f5a32b98d9 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -193,7 +193,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { io->clientRpc = rpcOpen(&rpcInit); if (io->clientRpc == NULL) { sError("failed to initialize RPC"); - return -1; + return terrno; } } @@ -214,7 +214,7 @@ static int32_t syncIOStartInternal(SSyncIO *io) { void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { sError("failed to start RPC server"); - return -1; + return terrno; } } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index c07f442c4a..59b7f4af2e 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -194,7 +194,11 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in int idx; int c; - (void)tdbBtcOpen(&btc, pBt, pTxn); + ret = tdbBtcOpen(&btc, pBt, pTxn); + if (ret) { + tdbError("tdb/btree-insert: btc open failed with ret: %d.", ret); + return ret; + } tdbTrace("tdb insert, btc: %p, pTxn: %p", &btc, pTxn); @@ -235,7 +239,11 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { int c; int ret; - (void)tdbBtcOpen(&btc, pBt, pTxn); + ret = tdbBtcOpen(&btc, pBt, pTxn); + if (ret) { + tdbError("tdb/btree-delete: btc open failed with ret: %d.", ret); + return ret; + } /* btc.coder.ofps = taosArrayInit(8, sizeof(SPage *)); // btc.coder.ofps = taosArrayInit(8, sizeof(SPgno)); @@ -337,7 +345,11 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL void *pTVal = NULL; SCellDecoder cd = {0}; - (void)tdbBtcOpen(&btc, pBt, NULL); + ret = tdbBtcOpen(&btc, pBt, NULL); + if (ret) { + tdbError("tdb/btree-pget: btc open failed with ret: %d.", ret); + return ret; + } tdbTrace("tdb pget, btc: %p", &btc); @@ -1660,12 +1672,14 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { if (pTxn == NULL) { TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); if (!pTxn) { + pBtc->pTxn = NULL; return terrno; } int32_t ret = tdbTxnOpen(pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); if (ret < 0) { tdbOsFree(pTxn); + pBtc->pTxn = NULL; return ret; } @@ -1881,7 +1895,6 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { if (cd.vLen > 0) { pVal = tdbRealloc(*ppVal, cd.vLen); if (pVal == NULL) { - tdbFree(pKey); return terrno; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index d885c38864..dacca1aea1 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -224,7 +224,10 @@ int tdbTbcOpen(TTB *pTb, TBC **ppTbc, TXN *pTxn) { return -1; } - (void)tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn); + if (tdbBtcOpen(&pTbc->btc, pTb->pBt, pTxn)) { + taosMemoryFree(pTbc); + return -1; + } *ppTbc = pTbc; return 0; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e66941244c..820075787f 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -148,7 +148,6 @@ typedef struct { STransSyncMsg* pSyncMsg; // for syncchronous with timeout API int64_t syncMsgRef; SCvtAddr cvtAddr; - bool setMaxRetry; int32_t retryMinInterval; int32_t retryMaxInterval; @@ -207,7 +206,7 @@ typedef struct { #pragma pack(pop) -typedef enum { Normal, Quit, Release, Register, Update } STransMsgType; +typedef enum { Normal, Quit, Release, Register, Update, FreeById } STransMsgType; typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } ConnStatus; #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) @@ -304,10 +303,10 @@ int32_t transClearBuffer(SConnBuffer* buf); int32_t transDestroyBuffer(SConnBuffer* buf); int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); bool transReadComplete(SConnBuffer* connBuf); -int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); -int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); +int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); -int transSetConnOption(uv_tcp_t* stream, int keepalive); +int32_t transSetConnOption(uv_tcp_t* stream, int keepalive); void transRefSrvHandle(void* handle); void transUnrefSrvHandle(void* handle); @@ -315,21 +314,24 @@ void transUnrefSrvHandle(void* handle); void transRefCliHandle(void* handle); void transUnrefCliHandle(void* handle); -int transReleaseCliHandle(void* handle); -int transReleaseSrvHandle(void* handle); +int32_t transReleaseCliHandle(void* handle); +int32_t transReleaseSrvHandle(void* handle); -int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); -int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated, +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); +int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs); -int transSendResponse(const STransMsg* msg); -int transRegisterMsg(const STransMsg* msg); -int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); +int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); +int32_t transFreeConnById(void* shandle, int64_t transpointId); + +int32_t transSendResponse(const STransMsg* msg); +int32_t transRegisterMsg(const STransMsg* msg); +int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func); -int transSockInfo2Str(struct sockaddr* sockname, char* dst); +int32_t transSockInfo2Str(struct sockaddr* sockname, char* dst); -int64_t transAllocHandle(); +int32_t transAllocHandle(int64_t* refId); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 7853e25cff..703a4dde3e 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -58,6 +58,8 @@ typedef struct { int32_t failFastThreshold; int32_t failFastInterval; + int8_t notWaitAvaliableConn; // 1: no delay, 0: delay + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index fbcc74e8e1..8b99443a84 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -102,6 +102,8 @@ void* rpcOpen(const SRpcInit* pInit) { if (pRpc->timeToGetConn == 0) { pRpc->timeToGetConn = 10 * 1000; } + pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; + pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -163,38 +165,48 @@ void* rpcReallocCont(void* ptr, int64_t contLen) { return st + TRANS_MSG_OVERHEAD; } -int rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { +int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { return transSendRequest(shandle, pEpSet, pMsg, NULL); } -int rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { - return transSendRequest(shandle, pEpSet, pMsg, pCtx); -} -int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { - return transSendRecv(shandle, pEpSet, pMsg, pRsp); -} -int rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, - int32_t timeoutMs) { - return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0|| pRid == NULL) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); + } } -int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } +int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +} + +int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { + return transSendRecv(shandle, pEpSet, pMsg, pRsp); +} +int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +} +int32_t rpcFreeConnById(void* shandle, int64_t connId) { return transFreeConnById(shandle, connId); } + +int32_t rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); } void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); } void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); } -int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } -int rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } +int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } +int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } // client only -int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { +int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { // later return transSetDefaultAddr(thandle, ip, fqdn); } // server only int32_t rpcSetIpWhite(void* thandle, void* arg) { return transSetIpWhiteList(thandle, arg, NULL); } -void* rpcAllocHandle() { return (void*)transAllocHandle(); } +int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle(refId); } int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2e61f19af8..4b324e52c6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -213,8 +213,10 @@ static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd); static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd); static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); -static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, - cliHandleUpdate}; +static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd); + +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, + NULL, cliHandleUpdate, cliHandleFreeById}; /// static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, /// NULL,cliHandleUpdate}; @@ -660,7 +662,9 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { if (QUEUE_IS_EMPTY(&plist->conns)) { if (plist->list->numOfConn >= pTranInst->connLimitNum) { *exceed = true; + return NULL;; } + plist->list->numOfConn++; return NULL; } @@ -704,7 +708,7 @@ static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) { SMsgList* list = plist->list; if ((list)->numOfConn >= pTransInst->connLimitNum) { STraceId* trace = &(*pMsg)->msg.info.traceId; - if (pTransInst->noDelayFp != NULL && pTransInst->noDelayFp((*pMsg)->msg.msgType)) { + if (pTransInst->notWaitAvaliableConn || (pTransInst->noDelayFp != NULL && pTransInst->noDelayFp((*pMsg)->msg.msgType))) { tDebug("%s msg %s not to send, reason: %s", pTransInst->label, TMSG_INFO((*pMsg)->msg.msgType), tstrerror(TSDB_CODE_RPC_NETWORK_BUSY)); doNotifyApp(*pMsg, pThrd, TSDB_CODE_RPC_NETWORK_BUSY); @@ -899,10 +903,12 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { exh->handle = conn; exh->pThrd = conn->hostThrd; taosWUnLockLatch(&exh->latch); - + conn->refId = exh->refId; taosWUnLockLatch(&exh->latch); + tDebug("conn %p specified by %"PRId64"", conn, handle); + (void)transReleaseExHandle(transGetRefMgt(), handle); return 0; } @@ -1035,7 +1041,6 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { list->size--; } } - conn->list = NULL; (void)transReleaseExHandle(transGetRefMgt(), conn->refId); @@ -1075,8 +1080,11 @@ static void cliDestroy(uv_handle_t* handle) { (void)atomic_sub_fetch_32(&pThrd->connCount, 1); + if (conn->refId > 0) { (void)transReleaseExHandle(transGetRefMgt(), conn->refId); (void)transRemoveExHandle(transGetRefMgt(), conn->refId); + + } taosMemoryFree(conn->dstAddr); taosMemoryFree(conn->stream); @@ -1589,6 +1597,40 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } +static void cliHandleFreeById(SCliMsg* pMsg, SCliThrd* pThrd) { + int32_t code = 0; + int64_t refId = (int64_t)(pMsg->msg.info.handle); + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); + if (exh == NULL) { + tDebug("id %" PRId64 " already released", refId); + destroyCmsg(pMsg); + return; + } + + taosRLockLatch(&exh->latch); + SCliConn* conn = exh->handle; + taosRUnLockLatch(&exh->latch); + + if (conn == NULL || conn->refId != refId) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); + } + tDebug("do free conn %p by id %" PRId64 "", conn, refId); + + int32_t size = transQueueSize(&conn->cliMsgs); + if (size == 0) { + // already recv, and notify upper layer + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); + return; + } else { + while (T_REF_VAL_GET(conn) >= 1) transUnrefCliHandle(conn); + } + return; +_exception: + tDebug("already free conn %p by id %" PRId64"", conn, refId); + + (void)transReleaseExHandle(transGetRefMgt(), refId); + destroyCmsg(pMsg); +} SCliConn* cliGetConn(SCliMsg** pMsg, SCliThrd* pThrd, bool* ignore, char* addr) { STransConnCtx* pCtx = (*pMsg)->ctx; @@ -2759,7 +2801,7 @@ SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle); return pThrd; } -int transReleaseCliHandle(void* handle) { +int32_t transReleaseCliHandle(void* handle) { int32_t code = 0; SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle); if (pThrd == NULL) { @@ -2823,25 +2865,25 @@ static int32_t transInitMsg(void* shandle, const SEpSet* pEpSet, STransMsg* pReq cliMsg->type = Normal; cliMsg->refId = (int64_t)shandle; QUEUE_INIT(&cliMsg->seqq); + *pCliMsg = cliMsg; return 0; } -int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); - return TSDB_CODE_RPC_BROKEN_LINK; + return TSDB_CODE_RPC_MODULE_QUIT; } int32_t code = 0; int64_t handle = (int64_t)pReq->info.handle; SCliThrd* pThrd = transGetWorkThrd(pTransInst, handle); if (pThrd == NULL) { - transFreeMsg(pReq->pCont); - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return TSDB_CODE_RPC_BROKEN_LINK; + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception;); } + if (handle != 0) { SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); if (exh != NULL) { @@ -2849,26 +2891,27 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran if (exh->handle == NULL && exh->inited != 0) { SCliMsg* pCliMsg = NULL; code = transInitMsg(shandle, pEpSet, pReq, ctx, &pCliMsg); - ASSERT(code == 0); + if (code != 0) { + taosWUnLockLatch(&exh->latch); + (void)transReleaseExHandle(transGetRefMgt(), handle); + TAOS_CHECK_GOTO(code, NULL, _exception); + } QUEUE_PUSH(&exh->q, &pCliMsg->seqq); taosWUnLockLatch(&exh->latch); tDebug("msg refId: %" PRId64 "", handle); (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; + } else { + exh->inited = 1; + taosWUnLockLatch(&exh->latch); + (void)transReleaseExHandle(transGetRefMgt(), handle); } - exh->inited = 1; - taosWUnLockLatch(&exh->latch); - (void)transReleaseExHandle(transGetRefMgt(), handle); } } SCliMsg* pCliMsg = NULL; - code = transInitMsg(shandle, pEpSet, pReq, ctx, &pCliMsg); - if (code != 0) { - (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return code; - } + TAOS_CHECK_GOTO(transInitMsg(shandle, pEpSet, pReq, ctx, &pCliMsg), NULL, _exception); STraceId* trace = &pReq->info.traceId; tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, @@ -2880,13 +2923,63 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran } (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return 0; + +_exception: + transFreeMsg(pReq->pCont); + (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return code; +} +int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + if (transpointId == NULL) { + ASSERT(0); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = 0; + + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + + TAOS_CHECK_GOTO(transAllocHandle(transpointId), NULL, _exception); + + SCliThrd* pThrd = transGetWorkThrd(pTransInst, *transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception); + } + + SExHandle* exh = transAcquireExHandle(transGetRefMgt(), *transpointId); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + + pReq->info.handle = (void*)(*transpointId); + + SCliMsg* pCliMsg = NULL; + TAOS_CHECK_GOTO(transInitMsg(shandle, pEpSet, pReq, NULL, &pCliMsg), NULL, _exception); + + STraceId* trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); + if ((code = transAsyncSend(pThrd->asyncPool, &(pCliMsg->q))) != 0) { + destroyCmsg(pCliMsg); + (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + } + (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return 0; + +_exception: + transFreeMsg(pReq->pCont); + (void)transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return code; } -int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); - return TSDB_CODE_RPC_BROKEN_LINK; + return TSDB_CODE_RPC_MODULE_QUIT; } int32_t code = 0; @@ -2908,8 +3001,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs code = tsem_init(sem, 0, 0); if (code != 0) { taosMemoryFree(sem); - code = TAOS_SYSTEM_ERROR(errno); - TAOS_CHECK_GOTO(code, NULL, _RETURN1); + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _RETURN1); } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -3003,13 +3095,13 @@ _EXIT: taosMemoryFree(pSyncMsg); return code; } -int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, - int32_t timeoutMs) { +int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { int32_t code = 0; STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { transFreeMsg(pReq->pCont); - return TSDB_CODE_RPC_BROKEN_LINK; + return TSDB_CODE_RPC_MODULE_QUIT; } STransMsg* pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); @@ -3096,22 +3188,21 @@ _RETURN2: /* * **/ -int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { +int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { + if (ip == NULL || fqdn == NULL) return TSDB_CODE_INVALID_PARA; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); if (pTransInst == NULL) { - return TSDB_CODE_RPC_BROKEN_LINK; + return TSDB_CODE_RPC_MODULE_QUIT; } SCvtAddr cvtAddr = {0}; - if (ip != NULL && fqdn != NULL) { - tstrncpy(cvtAddr.ip, ip, sizeof(cvtAddr.ip)); - tstrncpy(cvtAddr.fqdn, fqdn, sizeof(cvtAddr.fqdn)); - cvtAddr.cvt = true; - } + tstrncpy(cvtAddr.ip, ip, sizeof(cvtAddr.ip)); + tstrncpy(cvtAddr.fqdn, fqdn, sizeof(cvtAddr.fqdn)); + cvtAddr.cvt = true; int32_t code = 0; - int8_t i = 0; - for (i = 0; i < pTransInst->numOfThreads; i++) { + for (int8_t i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); if (pCtx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -3136,7 +3227,9 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { if ((code = transAsyncSend(thrd->asyncPool, &(cliMsg->q))) != 0) { destroyCmsg(cliMsg); - code = (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + if (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT) { + code = TSDB_CODE_RPC_MODULE_QUIT; + } break; } } @@ -3145,7 +3238,7 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return code; } -int64_t transAllocHandle() { +int32_t transAllocHandle(int64_t* refId) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); if (exh == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3166,5 +3259,43 @@ int64_t transAllocHandle() { QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); tDebug("pre alloc refId %" PRId64 "", exh->refId); - return exh->refId; + *refId = exh->refId; + return 0; +} +int32_t transFreeConnById(void* shandle, int64_t transpointId) { + int32_t code = 0; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + if (transpointId == 0) { + tDebug("not free by refId:%"PRId64"", transpointId); + TAOS_CHECK_GOTO(0, NULL, _exception); + } + + SCliThrd* pThrd = transGetWorkThrdFromHandle(pTransInst, transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); + } + + SCliMsg* pCli = taosMemoryCalloc(1, sizeof(SCliMsg)); + if (pCli == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _exception); + } + pCli->type = FreeById; + + tDebug("release conn id %" PRId64 "", transpointId); + + STransMsg msg = {.info.handle = (void*)transpointId}; + pCli->msg = msg; + + code = transAsyncSend(pThrd->asyncPool, &pCli->q); + if (code != 0) { + taosMemoryFree(pCli); + TAOS_CHECK_GOTO(code, NULL, _exception); + } + +_exception: + transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); + return code; } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 9df0ddb6f3..148f4d4e9a 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -234,7 +234,7 @@ bool transReadComplete(SConnBuffer* connBuf) { return (p->left == 0 || p->invalid) ? true : false; } -int transSetConnOption(uv_tcp_t* stream, int keepalive) { +int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { #if defined(WINDOWS) || defined(DARWIN) #else return uv_tcp_keepalive(stream, 1, keepalive); @@ -745,8 +745,7 @@ int32_t transRemoveExHandle(int32_t refMgt, int64_t refId) { return taosRemoveRef(refMgt, refId); } -void* transAcquireExHandle(int32_t refMgt, int64_t refId) { - // acquire extern handle +void* transAcquireExHandle(int32_t refMgt, int64_t refId) { // acquire extern handle return (void*)taosAcquireRef(refMgt, refId); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 1e0d54eb5b..0202fbc599 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1707,7 +1707,7 @@ void transUnrefSrvHandle(void* handle) { } } -int transReleaseSrvHandle(void* handle) { +int32_t transReleaseSrvHandle(void* handle) { int32_t code = 0; SRpcHandleInfo* info = handle; SExHandle* exh = info->handle; @@ -1747,7 +1747,7 @@ _return2: return code; } -int transSendResponse(const STransMsg* msg) { +int32_t transSendResponse(const STransMsg* msg) { int32_t code = 0; if (msg->info.noResp) { @@ -1800,7 +1800,7 @@ _return2: rpcFreeCont(msg->pCont); return code; } -int transRegisterMsg(const STransMsg* msg) { +int32_t transRegisterMsg(const STransMsg* msg) { int32_t code = 0; SExHandle* exh = msg->info.handle; @@ -1891,4 +1891,4 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { return code; } -int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } +int32_t transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } diff --git a/source/libs/transport/test/cliBench.c b/source/libs/transport/test/cliBench.c index 8a5276b814..ec08f1baf0 100644 --- a/source/libs/transport/test/cliBench.c +++ b/source/libs/transport/test/cliBench.c @@ -160,7 +160,7 @@ int main(int argc, char *argv[]) { void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { tError("failed to initialize RPC"); - return -1; + return terrno; } tInfo("client is initialized"); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 72f0a41710..c6c314446d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -47,14 +47,15 @@ char tsAVX512Supported = 0; int32_t osDefaultInit() { int32_t code = TSDB_CODE_SUCCESS; - + taosSeedRand(taosSafeRand()); taosGetSystemLocale(tsLocale, tsCharset); taosGetSystemTimezone(tsTimezoneStr, &tsTimezone); - code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone); - if (code) { - return code; + if (strlen(tsTimezoneStr) > 0) { // ignore empty timezone + if ((code = taosSetSystemTimezone(tsTimezoneStr, tsTimezoneStr, &tsDaylight, &tsTimezone)) != TSDB_CODE_SUCCESS) + return code; } + taosGetSystemInfo(); // deadlock in query diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index f9ae6c2157..081ed46c9a 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -1075,14 +1075,14 @@ int32_t taosGetFqdn(char *fqdn) { freeaddrinfo(result); -#else +#elif WINDOWS struct addrinfo hints = {0}; struct addrinfo *result = NULL; hints.ai_flags = AI_CANONNAME; int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr, "failed to get fqdn, code:%d, reason:%s\n", ret, gai_strerror(ret)); + fprintf(stderr, "failed to get fqdn, code:%d, hostname:%s, reason:%s\n", ret, hostname, gai_strerror(ret)); return -1; } strcpy(fqdn, result->ai_canonname); diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index cf1fef71c2..3e37d12759 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -242,7 +242,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const return EINVAL; #else int32_t code = pthread_cond_timedwait(cond, mutex, abstime); - if (code) { + if (code && code != ETIMEDOUT) { terrno = TAOS_SYSTEM_ERROR(code); return terrno; } @@ -445,9 +445,8 @@ int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) { return EBUSY; #else int32_t code = pthread_mutex_trylock(mutex); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return terrno; + if (code && code != EBUSY) { + code = TAOS_SYSTEM_ERROR(code); } return code; #endif @@ -816,9 +815,8 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) { return pthread_mutex_trylock((pthread_mutex_t *)lock); #else int32_t code = pthread_spin_trylock((pthread_spinlock_t *)lock); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - return code; + if (code && code != EBUSY) { + code = TAOS_SYSTEM_ERROR(code); } return code; #endif diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index ef45b773e8..c801347fc2 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -826,7 +826,7 @@ int32_t taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, i terrno = TAOS_SYSTEM_ERROR(errno); return terrno; } - + tzset(); int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); *tsTimezone = tz; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index e0d38df5c8..37667e2975 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -95,11 +95,12 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { tsize = (newSize == tsize) ? (tsize + 2) : newSize; } - pArray->pData = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); - if (pArray->pData == NULL) { + char* p = taosMemoryRealloc(pArray->pData, tsize * pArray->elemSize); + if (p == NULL) { return terrno; } + pArray->pData = p; pArray->capacity = tsize; } return 0; diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 4cb48bffe5..670a70a309 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1208,20 +1208,28 @@ typedef struct UsingRegex { regex_t pRegex; int32_t lastUsedTime; } UsingRegex; +typedef UsingRegex* HashRegexPtr; typedef struct RegexCache { SHashObj *regexHash; void *regexCacheTmr; void *timer; + SRWLatch mutex; + bool exit; } RegexCache; static RegexCache sRegexCache; #define MAX_REGEX_CACHE_SIZE 20 #define REGEX_CACHE_CLEAR_TIME 30 static void checkRegexCache(void* param, void* tmrId) { + int32_t code = 0; + taosRLockLatch(&sRegexCache.mutex); + if(sRegexCache.exit) { + goto _exit; + } (void)taosTmrReset(checkRegexCache, REGEX_CACHE_CLEAR_TIME * 1000, param, sRegexCache.regexCacheTmr, &tmrId); if (taosHashGetSize(sRegexCache.regexHash) < MAX_REGEX_CACHE_SIZE) { - return; + goto _exit; } if (taosHashGetSize(sRegexCache.regexHash) >= MAX_REGEX_CACHE_SIZE) { @@ -1235,6 +1243,8 @@ static void checkRegexCache(void* param, void* tmrId) { ppUsingRegex = taosHashIterate(sRegexCache.regexHash, ppUsingRegex); } } +_exit: + taosRUnLockLatch(&sRegexCache.mutex); } void regexCacheFree(void *ppUsingRegex) { @@ -1246,30 +1256,35 @@ int32_t InitRegexCache() { sRegexCache.regexHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (sRegexCache.regexHash == NULL) { uError("failed to create RegexCache"); - return -1; + return terrno; } taosHashSetFreeFp(sRegexCache.regexHash, regexCacheFree); sRegexCache.regexCacheTmr = taosTmrInit(0, 0, 0, "REGEXCACHE"); if (sRegexCache.regexCacheTmr == NULL) { uError("failed to create regex cache check timer"); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + return terrno; } + sRegexCache.exit = false; + taosInitRWLatch(&sRegexCache.mutex); sRegexCache.timer = taosTmrStart(checkRegexCache, REGEX_CACHE_CLEAR_TIME * 1000, NULL, sRegexCache.regexCacheTmr); if (sRegexCache.timer == NULL) { uError("failed to start regex cache timer"); - return -1; + return terrno; } - return 0; + return TSDB_CODE_SUCCESS; } void DestroyRegexCache(){ + int32_t code = 0; uInfo("[regex cache] destory regex cache"); (void)taosTmrStopA(&sRegexCache.timer); + taosWLockLatch(&sRegexCache.mutex); + sRegexCache.exit = true; taosHashCleanup(sRegexCache.regexHash); taosTmrCleanUp(sRegexCache.regexCacheTmr); + taosWUnLockLatch(&sRegexCache.mutex); } int32_t checkRegexPattern(const char *pPattern) { @@ -1290,18 +1305,17 @@ int32_t checkRegexPattern(const char *pPattern) { return TSDB_CODE_SUCCESS; } -static UsingRegex **getRegComp(const char *pPattern) { - UsingRegex **ppUsingRegex = (UsingRegex **)taosHashAcquire(sRegexCache.regexHash, pPattern, strlen(pPattern)); +int32_t getRegComp(const char *pPattern, HashRegexPtr **regexRet) { + HashRegexPtr* ppUsingRegex = (HashRegexPtr*)taosHashAcquire(sRegexCache.regexHash, pPattern, strlen(pPattern)); if (ppUsingRegex != NULL) { (*ppUsingRegex)->lastUsedTime = taosGetTimestampSec(); - return ppUsingRegex; + *regexRet = ppUsingRegex; + return TSDB_CODE_SUCCESS; } - UsingRegex *pUsingRegex = taosMemoryMalloc(sizeof(UsingRegex)); if (pUsingRegex == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; uError("Failed to Malloc when compile regex pattern %s.", pPattern); - return NULL; + return terrno; } int32_t cflags = REG_EXTENDED; int32_t ret = regcomp(&pUsingRegex->pRegex, pPattern, cflags); @@ -1310,8 +1324,7 @@ static UsingRegex **getRegComp(const char *pPattern) { (void)regerror(ret, &pUsingRegex->pRegex, msgbuf, tListLen(msgbuf)); uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); taosMemoryFree(pUsingRegex); - terrno = TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR; - return NULL; + return TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR; } while (true) { @@ -1319,8 +1332,9 @@ static UsingRegex **getRegComp(const char *pPattern) { if (code != 0 && code != TSDB_CODE_DUP_KEY) { regexCacheFree(&pUsingRegex); uError("Failed to put regex pattern %s into cache, exception internal error.", pPattern); - terrno = code; - return NULL; + return code; + } else if (code == TSDB_CODE_DUP_KEY) { + terrno = 0; } ppUsingRegex = (UsingRegex **)taosHashAcquire(sRegexCache.regexHash, pPattern, strlen(pPattern)); if (ppUsingRegex) { @@ -1334,27 +1348,68 @@ static UsingRegex **getRegComp(const char *pPattern) { } } pUsingRegex->lastUsedTime = taosGetTimestampSec(); - return ppUsingRegex; + *regexRet = ppUsingRegex; + return TSDB_CODE_SUCCESS; } void releaseRegComp(UsingRegex **regex){ taosHashRelease(sRegexCache.regexHash, regex); } +static threadlocal UsingRegex ** ppUsingRegex; +static threadlocal regex_t * pRegex; +static threadlocal char *pOldPattern = NULL; +void DestoryThreadLocalRegComp() { + if (NULL != pOldPattern) { + releaseRegComp(ppUsingRegex); + taosMemoryFree(pOldPattern); + ppUsingRegex = NULL; + pRegex = NULL; + pOldPattern = NULL; + } +} + +int32_t threadGetRegComp(regex_t **regex, const char *pPattern) { + if (NULL != pOldPattern) { + if (strcmp(pOldPattern, pPattern) == 0) { + *regex = pRegex; + return 0; + } else { + DestoryThreadLocalRegComp(); + } + } + + HashRegexPtr *ppRegex = NULL; + int32_t code = getRegComp(pPattern, &ppRegex); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pOldPattern = taosStrdup(pPattern); + if (NULL == pOldPattern) { + uError("Failed to Malloc when compile regex pattern %s.", pPattern); + return terrno; + } + ppUsingRegex = ppRegex; + pRegex = &((*ppUsingRegex)->pRegex); + *regex = &(*ppRegex)->pRegex; + return 0; +} + static int32_t doExecRegexMatch(const char *pString, const char *pPattern) { int32_t ret = 0; char msgbuf[256] = {0}; - UsingRegex **pUsingRegex = getRegComp(pPattern); - if (pUsingRegex == NULL) { - return 1; + + regex_t *regex = NULL; + ret = threadGetRegComp(®ex, pPattern); + if (ret != 0) { + return ret; } regmatch_t pmatch[1]; - ret = regexec(&(*pUsingRegex)->pRegex, pString, 1, pmatch, 0); - releaseRegComp(pUsingRegex); + ret = regexec(regex, pString, 1, pmatch, 0); if (ret != 0 && ret != REG_NOMATCH) { terrno = TSDB_CODE_PAR_REGULAR_EXPRESSION_ERROR; - (void)regerror(ret, &(*pUsingRegex)->pRegex, msgbuf, sizeof(msgbuf)); + (void)regerror(ret, regex, msgbuf, sizeof(msgbuf)); uDebug("Failed to match %s with pattern %s, reason %s", pString, pPattern, msgbuf) } @@ -1365,8 +1420,7 @@ int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); char *pattern = taosMemoryMalloc(sz + 1); if (NULL == pattern) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return 1; + return 1; // terrno has been set } (void)memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); @@ -1376,8 +1430,7 @@ int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { char *str = taosMemoryMalloc(sz + 1); if (NULL == str) { taosMemoryFree(pattern); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return 1; + return 1; // terrno has been set } (void)memcpy(str, varDataVal(pLeft), sz); @@ -1395,14 +1448,13 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { size_t len = varDataLen(pPattern); char *pattern = taosMemoryMalloc(len + 1); if (NULL == pattern) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return 1; + return 1; // terrno has been set } int convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pPattern), len, pattern); if (convertLen < 0) { taosMemoryFree(pattern); - return (terrno = TSDB_CODE_APP_ERROR); + return 1; // terrno has been set } pattern[convertLen] = 0; @@ -1411,15 +1463,14 @@ int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { char *str = taosMemoryMalloc(len + 1); if (NULL == str) { taosMemoryFree(pattern); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return 1; + return 1; // terrno has been set } convertLen = taosUcs4ToMbs((TdUcs4 *)varDataVal(pString), len, str); if (convertLen < 0) { taosMemoryFree(str); taosMemoryFree(pattern); - return (terrno = TSDB_CODE_APP_ERROR); + return 1; // terrno has been set } str[convertLen] = 0; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 87551e28e0..b14b0823a3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -244,6 +244,12 @@ static int32_t doSetConf(SConfigItem *pItem, const char *value, ECfgSrcType styp static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { TAOS_CHECK_RETURN(doSetConf(pItem, value, stype)); + if (strlen(value) == 0) { + uError("cfg:%s, type:%s src:%s, value:%s, skip to set timezone", pItem->name, cfgDtypeStr(pItem->dtype), + cfgStypeStr(stype), value); + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + TAOS_CHECK_RETURN(osSetTimezone(value)); TAOS_RETURN(TSDB_CODE_SUCCESS); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 2daedfdf32..5c1909b16e 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -624,6 +624,9 @@ void taosHashTableResize(SHashObj *pHashObj) { size_t inc = newCapacity - pHashObj->capacity; void *p = taosMemoryCalloc(inc, sizeof(SHashEntry)); + if (p == NULL) { + return; + } for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1946a0a274..86dc767adc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -567,7 +567,7 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } -void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { +void taosPrintLog(const char *flags, int32_t level, int32_t dflag, const char *format, ...) { if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; char buffer[LOG_MAX_LINE_BUFFER_SIZE]; @@ -590,7 +590,7 @@ void taosPrintLog(const char *flags, ELogLevel level, int32_t dflag, const char } } -void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, const char *format, ...) { +void taosPrintLongString(const char *flags, int32_t level, int32_t dflag, const char *format, ...) { if (!osLogSpaceAvailable()) return; if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 0e24d54565..7973a84593 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -39,7 +39,7 @@ int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, (SMultiwayMergeTreeInfo*)taosMemoryCalloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); + return terrno; } pTreeInfo->pNode = (STreeNode*)(((char*)pTreeInfo) + sizeof(SMultiwayMergeTreeInfo)); diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 76cf067842..a1fbe3db2a 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -352,9 +352,8 @@ static SPageInfo* getPageInfoFromPayload(void* page) { int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir) { - *pBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); - - SDiskbasedBuf* pPBuf = *pBuf; + *pBuf = NULL; + SDiskbasedBuf* pPBuf = taosMemoryCalloc(1, sizeof(SDiskbasedBuf)); if (pPBuf == NULL) { goto _error; } @@ -368,6 +367,9 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->fileSize = 0; pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); pPBuf->freePgList = tdListNew(POINTER_BYTES); + if (pPBuf->pFree == NULL || pPBuf->freePgList == NULL) { + goto _error; + } // at least more than 2 pages must be in memory if (inMemBufSize < pagesize * 2) { @@ -394,13 +396,19 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->prefix = (char*)dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); + if (pPBuf->emptyDummyIdList == NULL) { + goto _error; + } // qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, // pPBuf->pageSize, pPBuf->inMemPages, pPBuf->path); + *pBuf = pPBuf; return TSDB_CODE_SUCCESS; + _error: destroyDiskbasedBuf(pPBuf); + *pBuf = NULL; return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/util/src/tpcre2.c b/source/util/src/tpcre2.c index 52991c58b8..ba9bd51510 100644 --- a/source/util/src/tpcre2.c +++ b/source/util/src/tpcre2.c @@ -5,14 +5,24 @@ int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const ch int errorcode; PCRE2_SIZE erroroffset; - *ppRegex = pcre2_compile((PCRE2_SPTR8)pattern, PCRE2_ZERO_TERMINATED, options, &errorcode, &erroroffset, NULL); - if (*ppRegex == NULL) { + pcre2_code* pRegex = NULL; + pcre2_match_data* pMatchData = NULL; + + pRegex = pcre2_compile((PCRE2_SPTR8)pattern, PCRE2_ZERO_TERMINATED, options, &errorcode, &erroroffset, NULL); + if (pRegex == NULL) { PCRE2_UCHAR buffer[256]; (void)pcre2_get_error_message(errorcode, buffer, sizeof(buffer)); - return 1; + return -1; } - *ppMatchData = pcre2_match_data_create_from_pattern(*ppRegex, NULL); + pMatchData = pcre2_match_data_create_from_pattern(pRegex, NULL); + if (pMatchData == NULL) { + pcre2_code_free(pRegex); + return -1; + } + + *ppRegex = pRegex; + *ppMatchData = pMatchData; return 0; } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 258d53c335..edf5b0b970 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -106,6 +106,7 @@ static void *tQWorkerThreadFp(SQueueWorker *worker) { } destroyThreadLocalGeosCtx(); + DestoryThreadLocalRegComp(); return NULL; } @@ -237,6 +238,7 @@ static void *tAutoQWorkerThreadFp(SQueueWorker *worker) { taosUpdateItemSize(qinfo.queue, 1); } + DestoryThreadLocalRegComp(); return NULL; } @@ -664,6 +666,7 @@ static void *tQueryAutoQWorkerThreadFp(SQueryAutoQWorker *worker) { } destroyThreadLocalGeosCtx(); + DestoryThreadLocalRegComp(); return NULL; } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index e8aabfe338..0d8774ba41 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -119,6 +119,13 @@ add_test( COMMAND bufferTest ) +add_executable(regexTest "regexTest.cpp") +target_link_libraries(regexTest os util gtest_main ) +add_test( + NAME regexTest + COMMAND regexTest +) + #add_executable(decompressTest "decompressTest.cpp") #target_link_libraries(decompressTest os util common gtest_main) #add_test( diff --git a/source/util/test/regexTest.cpp b/source/util/test/regexTest.cpp new file mode 100644 index 0000000000..5fe3701700 --- /dev/null +++ b/source/util/test/regexTest.cpp @@ -0,0 +1,344 @@ + +#include +#include +#include +#include +#include +#include "os.h" +#include "tutil.h" +#include "regex.h" +#include "osDef.h" +#include "tcompare.h" + +extern "C" { + typedef struct UsingRegex UsingRegex; + typedef struct HashRegexPtr HashRegexPtr; + int32_t getRegComp(const char *pPattern, HashRegexPtr **regexRet); + int32_t threadGetRegComp(regex_t **regex, const char *pPattern); +} + +class regexTest { + public: + regexTest() { (void)InitRegexCache(); } + ~regexTest() { (void)DestroyRegexCache(); } +}; +static regexTest test; + +static threadlocal regex_t pRegex; +static threadlocal char *pOldPattern = NULL; + +void DestoryThreadLocalRegComp1() { + if (NULL != pOldPattern) { + regfree(&pRegex); + taosMemoryFree(pOldPattern); + pOldPattern = NULL; + } +} + +static regex_t *threadGetRegComp1(const char *pPattern) { + if (NULL != pOldPattern) { + if( strcmp(pOldPattern, pPattern) == 0) { + return &pRegex; + } else { + DestoryThreadLocalRegComp1(); + } + } + pOldPattern = (char*)taosMemoryMalloc(strlen(pPattern) + 1); + if (NULL == pOldPattern) { + uError("Failed to Malloc when compile regex pattern %s.", pPattern); + return NULL; + } + strcpy(pOldPattern, pPattern); + int32_t cflags = REG_EXTENDED; + int32_t ret = regcomp(&pRegex, pPattern, cflags); + if (ret != 0) { + char msgbuf[256] = {0}; + regerror(ret, &pRegex, msgbuf, tListLen(msgbuf)); + uError("Failed to compile regex pattern %s. reason %s", pPattern, msgbuf); + DestoryThreadLocalRegComp1(); + return NULL; + } + return &pRegex; +} + +TEST(testCase, regexCacheTest1) { + int times = 100000; + char s1[] = "abc"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("%s regex(current) %d times:%" PRIu64 " us.\n", s1, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = threadGetRegComp1(s1); + } + uint64_t t3 = taosGetTimestampUs(); + + printf("%s regex(before) %d times:%" PRIu64 " us.\n", s1, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + } + t3 = taosGetTimestampUs(); + + printf("%s regex(new) %d times:%" PRIu64 " us.\n", s1, times, t3 - t2); +} + +TEST(testCase, regexCacheTest2) { + int times = 100000; + char s1[] = "abc%*"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("%s regex(current) %d times:%" PRIu64 " us.\n", s1, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = threadGetRegComp1(s1); + } + uint64_t t3 = taosGetTimestampUs(); + + printf("%s regex(before) %d times:%" PRIu64 " us.\n", s1, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + } + t3 = taosGetTimestampUs(); + + printf("%s regex(new) %d times:%" PRIu64 " us.\n", s1, times, t3 - t2); +} + +TEST(testCase, regexCacheTest3) { + int times = 100000; + char s1[] = "abc%*"; + char s2[] = "abc"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn regex(current) %d times:%" PRIu64 " us.\n", s1, s2, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = threadGetRegComp1(s1); + rex = threadGetRegComp1(s2); + } + uint64_t t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn regex(before) %d times:%" PRIu64 " us.\n", s1, s2, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for(int i = 0; i < times; i++) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + (void)threadGetRegComp(&rex, s2); + } + t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn regex(new) %d times:%" PRIu64 " us.\n", s1, s2, times, t3 - t2); +} + +TEST(testCase, regexCacheTest4) { + int times = 100; + int count = 1000; + char s1[] = "abc%*"; + char s2[] = "abc"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s2, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s2; + } + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(current) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s2); + } + } + uint64_t t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(before) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s2); + } + } + t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(new) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); +} + +// It is not a good idea to test this case, because it will take a long time. +/* +TEST(testCase, regexCacheTest5) { + int times = 10000; + int count = 10000; + char s1[] = "abc%*"; + char s2[] = "abc"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s2, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s2; + } + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(current) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s2); + } + } + uint64_t t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(before) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s2); + } + } + t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(new) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); +} + +TEST(testCase, regexCacheTest6) { + int times = 10000; + int count = 1000; + char s1[] = "abc%*"; + char s2[] = "abc"; + auto start = std::chrono::high_resolution_clock::now(); + + uint64_t t0 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s1, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s1; + } + } + for (int j = 0; j < count; ++j) { + HashRegexPtr* ret = NULL; + int32_t code = getRegComp(s2, &ret); + if (code != 0) { + FAIL() << "Failed to compile regex pattern " << s2; + } + } + } + uint64_t t1 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(current) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t1 - t0); + + uint64_t t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = threadGetRegComp1(s2); + } + } + uint64_t t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(before) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); + + t2 = taosGetTimestampUs(); + for (int i = 0; i < times; i++) { + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s1); + } + for (int j = 0; j < count; ++j) { + regex_t* rex = NULL; + (void)threadGetRegComp(&rex, s2); + } + } + t3 = taosGetTimestampUs(); + + printf("'%s' and '%s' take place by turn(per %d count) regex(new) %d times:%" PRIu64 " us.\n", s1, s2, count, times, t3 - t2); +} +*/ diff --git a/tests/army/cluster/incSnapshot.py b/tests/army/cluster/incSnapshot.py index dfd8d95c9c..1028781fcb 100644 --- a/tests/army/cluster/incSnapshot.py +++ b/tests/army/cluster/incSnapshot.py @@ -15,7 +15,7 @@ from frame import * from frame.autogen import * # from frame.server.dnodes import * # from frame.server.cluster import * - +from frame.clusterCommonCheck import * class TDTestCase(TBase): updatecfgDict = { @@ -39,29 +39,12 @@ class TDTestCase(TBase): autoGen.insert_data(1000) tdSql.execute(f"flush database {self.db}") sc.dnodeStop(3) - # clusterDnodes.stoptaosd(1) - # clusterDnodes.starttaosd(3) - # time.sleep(5) - # clusterDnodes.stoptaosd(2) - # clusterDnodes.starttaosd(1) - # time.sleep(5) autoGen.insert_data(5000, True) self.flushDb(True) # wait flush operation over time.sleep(5) - # sql = 'show vnodes;' - # while True: - # bFinish = True - # param_list = tdSql.query(sql, row_tag=True) - # for param in param_list: - # if param[3] == 'leading' or param[3] == 'following': - # bFinish = False - # break - # if bFinish: - # break - self.snapshotAgg() - time.sleep(10) + self.snapshotAgg() sc.dnodeStopAll() for i in range(1, 4): path = clusterDnodes.getDnodeDir(i) @@ -75,19 +58,7 @@ class TDTestCase(TBase): sc.dnodeStart(2) sc.dnodeStart(3) sql = "show vnodes;" - time.sleep(10) - while True: - bFinish = True - param_list = tdSql.query(sql, row_tag=True) - for param in param_list: - if param[3] == 'offline': - tdLog.exit( - "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) - if param[3] == 'leading' or param[3] == 'following': - bFinish = False - break - if bFinish: - break + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=3,db_name=f"{self.db}",count_number=60) self.timestamp_step = 1000 self.insert_rows = 6000 diff --git a/tests/army/frame/clusterCommonCheck.py b/tests/army/frame/clusterCommonCheck.py new file mode 100644 index 0000000000..cca70a88b7 --- /dev/null +++ b/tests/army/frame/clusterCommonCheck.py @@ -0,0 +1,277 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from collections import defaultdict +import random +import string +import threading +import requests +import time +# import socketfrom + +import taos +from frame.log import * +from frame.sql import * +from frame.cases import * +from frame.server.dnodes import * +from frame.common import * + +# class actionType(Enum): +# CREATE_DATABASE = 0 +# CREATE_STABLE = 1 +# CREATE_CTABLE = 2 +# INSERT_DATA = 3 + +class ClusterComCheck: + def init(self, conn, logSql=False): + tdSql.init(conn.cursor()) + # tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkDnodes(self,dnodeNumbers, timeout=100): + count=0 + # print(tdSql) + while count < timeout: + tdSql.query("select * from information_schema.ins_dnodes") + # tdLog.debug(tdSql.res) + status=0 + for i in range(dnodeNumbers): + if tdSql.res[i][4] == "ready": + status+=1 + # tdLog.info(status) + + if status == dnodeNumbers: + tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within %ds! " % (dnodeNumbers, count+1)) + return True + time.sleep(1) + count+=1 + + else: + tdSql.query("select * from information_schema.ins_dnodes") + tdLog.debug(tdSql.res) + tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within %ds ! "% (dnodeNumbers, timeout)) + + def checkDbRows(self,dbNumbers): + dbNumbers=int(dbNumbers) + count=0 + while count < 5: + tdSql.query("select * from information_schema.ins_databases where name!='collectd' ;") + count+=1 + if tdSql.checkRows(dbNumbers+2): + tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2)) + return True + else: + continue + else : + tdLog.debug(tdSql.res) + tdLog.exit("we find %d databases but expect %d in clusters! " %(tdSql.queryRows,dbNumbers)) + + def checkDb(self,dbNumbers,restartNumber,dbNameIndex, timeout=100): + count=0 + alldbNumbers=(dbNumbers*restartNumber)+2 + while count < timeout: + query_status=0 + for j in range(dbNumbers): + for i in range(alldbNumbers): + tdSql.query("select * from information_schema.ins_databases;") + if "%s_%d"%(dbNameIndex,j) == tdSql.res[i][0] : + if tdSql.res[i][15] == "ready": + query_status+=1 + tdLog.debug("check %s_%d that status is ready "%(dbNameIndex,j)) + else: + sleep(1) + continue + # print(query_status) + if query_status == dbNumbers: + tdLog.success(" check %d database and all databases are ready within %ds! " %(dbNumbers,count+1)) + return True + count+=1 + + else: + tdLog.debug(tdSql.res) + tdLog.debug("query status is %d"%query_status) + tdLog.exit("database is not ready within %ds"%(timeout+1)) + + def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,): + tdSql.execute("use %s"%dbname) + tdSql.query("show %s.stables"%dbname) + tdSql.checkRows(stableCount) + tdSql.query("show %s.tables"%dbname) + tdSql.checkRows(CtableCount) + for i in range(stableCount): + tdSql.query("select count(*) from %s%d"%(stbname,i)) + tdSql.checkData(0,0,rowsPerSTable) + return + + def checkMnodeStatus(self,mnodeNums): + self.mnodeNums=int(mnodeNums) + # self.leaderDnode=int(leaderDnode) + tdLog.debug("start to check status of mnodes") + count=0 + + while count < 10: + time.sleep(1) + tdSql.query("select * from information_schema.ins_mnodes;") + if tdSql.checkRows(self.mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + + if self.mnodeNums == 1: + if tdSql.res[0][2]== 'leader' and tdSql.res[0][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + elif self.mnodeNums == 3 : + if tdSql.res[0][2]=='leader' and tdSql.res[0][3]== 'ready' : + if tdSql.res[1][2]=='follower' and tdSql.res[1][3]== 'ready' : + if tdSql.res[2][2]=='follower' and tdSql.res[2][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.res[1][2]=='leader' and tdSql.res[1][3]== 'ready' : + if tdSql.res[0][2]=='follower' and tdSql.res[0][3]== 'ready' : + if tdSql.res[2][2]=='follower' and tdSql.res[2][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.res[2][2]=='leader' and tdSql.res[2][3]== 'ready' : + if tdSql.res[0][2]=='follower' and tdSql.res[0][3]== 'ready' : + if tdSql.res[1][2]=='follower' and tdSql.res[1][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + elif self.mnodeNums == 2 : + if tdSql.res[0][2]=='leader' and tdSql.res[0][3]== 'ready' : + if tdSql.res[1][2]=='follower' and tdSql.res[1][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.res[1][2]=='leader' and tdSql.res[1][3]== 'ready' : + if tdSql.res[0][2]=='follower' and tdSql.res[0][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + else: + tdLog.debug(tdSql.res) + tdLog.exit("cluster of %d mnodes is not ready in 10s " %self.mnodeNums) + + + + + def check3mnodeoff(self,offlineDnodeNo,mnodeNums=3): + count=0 + while count < 10: + time.sleep(1) + tdSql.query("select * from information_schema.ins_mnodes;") + if tdSql.checkRows(mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + else: + tdLog.exit("mnode number is correct") + if offlineDnodeNo == 1: + if tdSql.res[0][2]=='offline' : + if tdSql.res[1][2]=='leader': + if tdSql.res[2][2]=='follower': + tdLog.success("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.res[1][2]=='follower': + if tdSql.res[2][2]=='leader': + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + elif offlineDnodeNo == 2: + if tdSql.res[1][2]=='offline' : + if tdSql.res[0][2]=='leader': + if tdSql.res[2][2]=='follower': + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.res[0][2]=='follower': + if tdSql.res[2][2]=='leader': + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + elif offlineDnodeNo == 3: + if tdSql.res[2][2]=='offline' : + if tdSql.res[0][2]=='leader': + if tdSql.res[1][2]=='follower': + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.res[0][2]=='follower': + if tdSql.res[1][2]=='leader': + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + else: + tdLog.debug(tdSql.res) + tdLog.exit(f"stop mnodes on dnode {offlineDnodeNo} failed in 10s ") + + def check3mnode2off(self,mnodeNums=3): + count=0 + while count < 10: + time.sleep(1) + tdSql.query("select * from information_schema.ins_mnodes;") + if tdSql.checkRows(mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + else: + tdLog.exit("mnode number is correct") + if tdSql.res[0][2]=='leader' : + if tdSql.res[1][2]=='offline': + if tdSql.res[2][2]=='offline': + tdLog.success("stop mnodes of follower on dnode successfully in 10s") + return True + count+=1 + else: + tdLog.debug(tdSql.res) + tdLog.exit("stop mnodes on dnode 2 or 3 failed in 10s") + + def check_vgroups_status(self,vgroup_numbers=2,db_replica=3,count_number=10,db_name="db"): + """ check vgroups status in 10s after db vgroups status is changed """ + vgroup_numbers = int(vgroup_numbers) + self.db_replica = int(db_replica) + tdLog.debug("start to check status of vgroups") + count=0 + last_number=vgroup_numbers-1 + while count < count_number: + time.sleep(1) + count+=1 + print("check vgroup count :", count) + tdSql.query(f"show {db_name}.vgroups;") + if tdSql.getRows() != vgroup_numbers : + continue + if self.db_replica == 1 : + if tdSql.res[0][4] == 'leader' and tdSql.res[last_number][4] == 'leader': + tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") + print("db replica :",tdSql.res[0][0]) + if tdSql.res[0][0] == db_replica: + tdLog.success(f"all vgroups with replica {self.db_replica} of {db_name} are leaders in {count} s") + return True + + elif self.db_replica == 3 : + vgroup_status_first=[tdSql.res[0][4],tdSql.res[0][6],tdSql.res[0][8]] + + vgroup_status_last=[tdSql.res[last_number][4],tdSql.res[last_number][6],tdSql.res[last_number][8]] + if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2: + if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2: + tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") + print("db replica :",tdSql.res[0][0]) + if tdSql.res[0][0] == db_replica: + tdLog.success(f"elections of {db_name}.vgroups with replica {self.db_replica} are ready in {count} s") + return True + else: + tdLog.debug(tdSql.res) + tdLog.notice(f"elections of {db_name} all vgroups with replica {self.db_replica} are failed in {count} s ") + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno) + tdLog.exit("%s(%d) failed " % args) + + + + + def close(self): + self.cursor.close() + +clusterComCheck = ClusterComCheck() diff --git a/tests/ci/scan_file_path.py b/tests/ci/scan_file_path.py index 03f2d6ee4f..aff94158b8 100644 --- a/tests/ci/scan_file_path.py +++ b/tests/ci/scan_file_path.py @@ -122,7 +122,7 @@ def scan_files_path(source_file_path): for file in files: if any(item in root for item in scan_dir_list): file_path = os.path.join(root, file) - if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): + if (file_path.endswith(".c") or file_name.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list): all_file_path.append(file_path) logger.info("Found %s files" % len(all_file_path)) @@ -134,7 +134,7 @@ def input_files(change_files): for line in file: file_name = line.strip() if any(dir_name in file_name for dir_name in scan_dir_list): - if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list): + if (file_name.endswith(".c") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list): if "enterprise" in file_name: file_name = os.path.join(TD_project_path, file_name) else: @@ -226,4 +226,4 @@ if __name__ == "__main__": logger.error(f"Scan failed,please check the log file:{scan_result_log}") for index, failed_result_file in enumerate(web_path): logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}") - exit(1) \ No newline at end of file + exit(1) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e3f5e54698..e70042001d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -340,7 +340,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroup.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupWal.py -N 3 -n 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py @@ -544,6 +544,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/limit.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/logical_operators.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/logical_operators.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py diff --git a/tests/script/sh/bit_and.c b/tests/script/sh/bit_and.c index f3bf71ce94..2cf2157e1c 100644 --- a/tests/script/sh/bit_and.c +++ b/tests/script/sh/bit_and.c @@ -8,13 +8,17 @@ DLL_EXPORT int32_t bit_and_init() { return 0; } DLL_EXPORT int32_t bit_and_destroy() { return 0; } DLL_EXPORT int32_t bit_and(SUdfDataBlock* block, SUdfColumn* resultCol) { + udfTrace("block:%p, processing begins, rows:%d cols:%d", block, block->numOfRows, block->numOfCols); + if (block->numOfCols < 2) { + udfError("block:%p, cols:%d needs to be greater than 2", block, block->numOfCols); return TSDB_CODE_UDF_INVALID_INPUT; } for (int32_t i = 0; i < block->numOfCols; ++i) { SUdfColumn* col = block->udfCols[i]; - if (!(col->colMeta.type == TSDB_DATA_TYPE_INT)) { + if (col->colMeta.type != TSDB_DATA_TYPE_INT) { + udfError("block:%p, col:%d type:%d should be int(%d)", block, i, col->colMeta.type, TSDB_DATA_TYPE_INT); return TSDB_CODE_UDF_INVALID_INPUT; } } @@ -23,25 +27,35 @@ DLL_EXPORT int32_t bit_and(SUdfDataBlock* block, SUdfColumn* resultCol) { for (int32_t i = 0; i < block->numOfRows; ++i) { if (udfColDataIsNull(block->udfCols[0], i)) { + udfTrace("block:%p, row:%d result is null since col:0 is null", block, i); udfColDataSetNull(resultCol, i); continue; } + int32_t result = *(int32_t*)udfColDataGetData(block->udfCols[0], i); - int j = 1; + udfTrace("block:%p, row:%d col:0 data:%d", block, i, result); + + int32_t j = 1; for (; j < block->numOfCols; ++j) { if (udfColDataIsNull(block->udfCols[j], i)) { + udfTrace("block:%p, row:%d result is null since col:%d is null", block, i, j); udfColDataSetNull(resultCol, i); break; } char* colData = udfColDataGetData(block->udfCols[j], i); result &= *(int32_t*)colData; + udfTrace("block:%p, row:%d col:%d data:%d", block, i, j, *(int32_t*)colData); } + if (j == block->numOfCols) { udfColDataSet(resultCol, i, (char*)&result, false); + udfTrace("block:%p, row:%d result is %d", block, i, result); } } + resultData->numOfRows = block->numOfRows; + udfTrace("block:%p, processing completed, rows:%d, cols:%d,", block, block->numOfRows, block->numOfCols); return TSDB_CODE_SUCCESS; } diff --git a/tests/system-test/2-query/interval_limit_opt.py b/tests/system-test/2-query/interval_limit_opt.py index aa1702fe3c..2f222d5b43 100644 --- a/tests/system-test/2-query/interval_limit_opt.py +++ b/tests/system-test/2-query/interval_limit_opt.py @@ -195,6 +195,10 @@ class TDTestCase: tdSql.checkData(1, 4, 2) tdSql.checkData(2, 4, 9) tdSql.checkData(3, 4, 9) + + sql = "SELECT _wstart, last(c1) FROM t6 INTERVAL(1w);" + tdSql.query(sql) + tdSql.checkRows(11) def test_partition_by_limit_no_agg(self): sql_template = 'select t1 from meters partition by t1 limit %d' diff --git a/tests/system-test/2-query/logical_operators.py b/tests/system-test/2-query/logical_operators.py new file mode 100644 index 0000000000..5a89de92ef --- /dev/null +++ b/tests/system-test/2-query/logical_operators.py @@ -0,0 +1,120 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + +DBNAME = "db" + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.batchNum = 5 + self.ts = 1537146000000 + + def run(self,dbname=DBNAME): + tdSql.prepare() + + tdSql.execute(f'''create table {dbname}.tb (ts timestamp, v int, f float, b varchar(8))''') + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:00:00', 1, 2.0, 't0')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:01:00', 11, 12.1, 't0')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:02:00', 21, 22.2, 't0')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:03:00', 31, 32.3, 't0')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:04:00', 41, 42.4, 't0')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:05:00', 51, 52.5, 't1')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:06:00', 61, 62.6, 't1')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:07:00', 71, 72.7, 't1')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:08:00', 81, 82.8, 't1')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:09:00', 91, 92.9, 't1')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:00:00',101,112.9, 't2')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:01:00',111,112.1, 't2')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:02:00',121,122.2, 't2')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:03:00',131,132.3, 't2')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:04:00',141,142.4, 't2')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:05:00',151,152.5, 't3')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:06:00',161,162.6, 't3')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:07:00',171,172.7, 't3')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:08:00',181,182.8, 't3')") + tdSql.execute(f"insert into {dbname}.tb values('2024-07-04 10:09:00',191,192.9, 't3')") + #test for operator and + tdSql.query('''select +`T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` > 0 + and `T_9048C6F41B2A45CE94FF3`.`f` > `T_9048C6F41B2A45CE94FF3`.`v` +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + #test for operator or + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` > 0 + or `T_9048C6F41B2A45CE94FF3`.`f` > `T_9048C6F41B2A45CE94FF3`.`v` +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + #test for operator in + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` in (1) +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000;''') + tdSql.checkRows(10) + #test for operator not + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + not `T_9048C6F41B2A45CE94FF3`.`v` > 0 +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + #test for operator between and + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` between 1 and 200 +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + #test for operator is null + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` is null +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + #test for operator is not null + tdSql.query('''select + `T_9048C6F41B2A45CE94FF3`.`ts` as `__fcol_0`, + `T_9048C6F41B2A45CE94FF3`.`v` as `__fcol_1`, + `T_9048C6F41B2A45CE94FF3`.`f` as `__fcol_2`, + `T_9048C6F41B2A45CE94FF3`.`b` as `__fcol_3`, + `T_9048C6F41B2A45CE94FF3`.`v` is not null +from `db`.`tb` as `T_9048C6F41B2A45CE94FF3` +limit 5000''') + tdSql.checkRows(10) + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/7-tmq/tmqParamsTest.py b/tests/system-test/7-tmq/tmqParamsTest.py index a323dff19e..c14c3fc7d1 100644 --- a/tests/system-test/7-tmq/tmqParamsTest.py +++ b/tests/system-test/7-tmq/tmqParamsTest.py @@ -104,7 +104,7 @@ class TDTestCase: stop_flag = 0 try: while True: - res = consumer.poll(1) + res = consumer.poll(3) tdSql.query('show consumers;') consumer_info = tdSql.queryResult[0][-1] if offset_value == "latest": diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py index 6ef28a4e77..b5105def37 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py @@ -52,7 +52,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 8987cf5251..1488e304cb 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -52,7 +52,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -121,7 +121,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py index bad9e09da5..f77fb53c85 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py @@ -121,7 +121,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index a9fb1c2d4b..979d75d558 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -52,7 +52,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -121,7 +121,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index 3965168fa7..7c8f56f40d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -54,7 +54,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py index d4c76c4f61..5ff2ca6e27 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py @@ -54,7 +54,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py index a5e61adc8d..9d89e3b1c0 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py @@ -56,7 +56,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index eb35ebc718..3c5f3ecb30 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -56,7 +56,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py index 5aa2054e96..b7cebb51e0 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb.py @@ -54,7 +54,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -123,7 +123,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 120, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py index a853489c3f..034197e0f9 100644 --- a/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py +++ b/tests/system-test/7-tmq/tmqVnodeTransform-db-removewal.py @@ -140,7 +140,7 @@ class TDTestCase: 'rowsPerTbl': 10000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 2, + 'pollDelay': 5, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -190,9 +190,6 @@ class TDTestCase: # redistribute vgroup self.redistributeVgroups() - tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tdLog.info("wait the consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index d30d88bb1c..d0e682cffb 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -496,6 +496,43 @@ class TDTestCase: consumer.close() print("consume_ts_4551 ok") + def consume_td_31283(self): + tdSql.execute(f'create database if not exists d31283') + tdSql.execute(f'use d31283') + + tdSql.execute(f'create topic topic_31283 with meta as database d31283') + consumer_dict = { + "group.id": "g1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "auto.offset.reset": "earliest", + "experimental.snapshot.enable": "true", + # "msg.enable.batchmeta": "true" + } + consumer = Consumer(consumer_dict) + + try: + consumer.subscribe(["topic_31283"]) + except TmqError: + tdLog.exit(f"subscribe error") + + tdSql.execute(f'create table stt(ts timestamp, i int) tags(t int)') + + hasData = False + try: + while True: + res = consumer.poll(1) + if not res: + break + hasData = True + finally: + consumer.close() + + if not hasData: + tdLog.exit(f"consume_td_31283 error") + + print("consume_td_31283 ok") + def consume_TS_5067_Test(self): tdSql.execute(f'create database if not exists d1 vgroups 1') tdSql.execute(f'use d1') @@ -632,6 +669,7 @@ class TDTestCase: self.consume_ts_4544() self.consume_ts_4551() self.consume_TS_4540_Test() + self.consume_td_31283() tdSql.prepare() self.checkWal1VgroupOnlyMeta() diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 2f6a5fa59b..49cfa3dff8 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -454,7 +454,7 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes) { taos_free_result(pRes); #else pRes = taos_query(pConn, - "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 into meters_summary as select " + "create stream meters_summary_s trigger at_once IGNORE EXPIRED 0 fill_history 1 into meters_summary as select " "_wstart, max(current) as current, " "groupid, location from meters partition by groupid, location interval(10m)"); if (taos_errno(pRes) != 0) { @@ -633,7 +633,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } int32_t cnt = 0; while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); if (tmqmessage) { cnt++; msg_process(tmqmessage);