diff --git a/.gitmodules b/.gitmodules index 34a2da2894..9d649bcdd3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "tools/taosadapter"] path = tools/taosadapter url = https://github.com/taosdata/taosadapter.git +[submodule "tools/taosws-rs"] + path = tools/taosws-rs + url = https://github.com/taosdata/taosws-rs.git diff --git a/README-CN.md b/README-CN.md index c78050c7bc..39d979c123 100644 --- a/README-CN.md +++ b/README-CN.md @@ -1,38 +1,59 @@ +

+

+ + TDengine + +

+

+ [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine) [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) [![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine) -[![TDengine](TDenginelogo.png)](https://www.taosdata.com) - -简体中文 | [English](./README.md) +简体中文 | [English](README.md) | 很多职位正在热招中，请看[这里](https://www.taosdata.com/cn/careers/) # TDengine 简介 -TDengine是涛思数据专为物联网、车联网、工业互联网、IT运维等设计和优化的大数据平台。除核心的快10倍以上的时序数据库功能外，还提供缓存、数据订阅、流式计算等功能，最大程度减少研发和运维的复杂度，且核心代码，包括集群功能全部开源（开源协议，AGPL v3.0）。 +TDengine 是一款高性能、分布式、支持 SQL 的时序数据库（Time-Series Database）。而且除时序数据库功能外，它还提供缓存、数据订阅、流式计算等功能，最大程度减少研发和运维的复杂度，且核心代码，包括集群功能全部开源（开源协议，AGPL v3.0）。与其他时序数据数据库相比，TDengine 有以下特点： -- 10 倍以上性能提升。定义了创新的数据存储结构，单核每秒就能处理至少2万次请求，插入数百万个数据点，读出一千万以上数据点，比现有通用数据库快了十倍以上。 -- 硬件或云服务成本降至1/5。由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。 -- 全栈时序数据处理引擎。将数据库、消息队列、缓存、流式计算等功能融合一起，应用无需再集成Kafka/Redis/HBase/Spark等软件，大幅降低应用开发和维护成本。 -- 强大的分析功能。无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell/Python/R/Matlab随时进行。 -- 与第三方工具无缝连接。不用一行代码，即可与Telegraf, Grafana, EMQ X, Prometheus, Matlab, R集成。后续还将支持MQTT, OPC, Hadoop，Spark等, BI工具也将无缝连接。 -- 零运维成本、零学习成本。安装、集群一秒搞定，无需分库分表，实时备份。标准SQL，支持JDBC,RESTful，支持Python/Java/C/C++/Go/Node.JS, 与MySQL相似，零学习成本。 +- **高性能**：通过创新的存储引擎设计，无论是数据写入还是查询，TDengine 的性能比通用数据库快 10 倍以上，也远超其他时序数据库，而且存储空间也大为节省。 + +- **分布式**：通过原生分布式的设计，TDengine 提供了水平扩展的能力，只需要增加节点就能获得更强的数据处理能力，同时通过多副本机制保证了系统的高可用。 + +- **支持 SQL**：TDengine 采用 SQL 作为数据查询语言，减少学习和迁移成本，同时提供 SQL 扩展来处理时序数据特有的分析，而且支持方便灵活的 schemaless 数据写入。 + +- **All in One**：将数据库、消息队列、缓存、流式计算等功能融合一起，应用无需再集成 Kafka/Redis/HBase/Spark 等软件，大幅降低应用开发和维护成本。 + +- **零管理**：安装、集群几秒搞定，无任何依赖，不用分库分表，系统运行状态监测能与 Grafana 或其他运维工具无缝集成。 + +- **零学习成本**：采用 SQL 查询语言，支持 Python、Java、C/C++、Go、Rust、Node.js 等多种编程语言，与 MySQL 相似，零学习成本。 + +- **无缝集成**：不用一行代码，即可与 Telegraf、Grafana、EMQX、Prometheus、StatsD、collectd、Matlab、R 等第三方工具无缝集成。 + +- **互动 Console**: 通过命令行 console，不用编程，执行 SQL 语句就能做即席查询、各种数据库的操作、管理以及集群的维护. + +TDengine 可以广泛应用于物联网、工业互联网、车联网、IT 运维、能源、金融等领域，让大量设备、数据采集器每天产生的高达 TB 甚至 PB 级的数据能得到高效实时的处理，对业务的运行状态进行实时的监测、预警，从大数据中挖掘出商业价值。 # 文档 -TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是 [数据模型](https://www.taosdata.com/cn/documentation/architecture) 与 [数据建模](https://www.taosdata.com/cn/documentation/model)。除本文档之外，欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。 +TDengine 采用传统的关系数据库模型，您可以像使用关系型数据库 MySQL 一样来使用它。但由于引入了超级表，一个采集点一张表的概念，建议您在使用前仔细阅读一遍下面的文档，特别是 [数据模型](https://www.taosdata.com/cn/documentation/architecture) 与 [数据建模](https://www.taosdata.com/cn/documentation/model)。除本文档之外，欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。 # 构建 -TDengine目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、macOS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。本快速指南仅适用于通过源码安装。 +TDengine 目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后续会支持 Windows、macOS 等系统。客户端可以在 Windows 或 Linux 上安装和运行。任何 OS 的应用也可以选择 RESTful 接口连接服务器 taosd。CPU 支持 X64/ARM64/MIPS64/Alpha64，后续会支持 ARM32、RISC-V 等 CPU 架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。本快速指南仅适用于通过源码安装。 ## 安装工具 ### Ubuntu 16.04 及以上版本 & Debian： ```bash -sudo apt-get install -y gcc cmake build-essential git +sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` ### Ubuntu 14.04： @@ -56,10 +77,22 @@ sudo apt-get install -y openjdk-8-jdk sudo apt-get install -y maven ``` +#### 为 taos-tools 安装编译需要的软件 + +taosTools 是用于 TDengine 的辅助工具软件集合。目前它包含 taosBenchmark（曾命名为 taosdemo）和 taosdump 两个软件。 + +默认 TDengine 编译不包含 taosTools。您可以在编译 TDengine 时使用`cmake .. -DBUILD_TOOLS=true` 来同时编译 taosTools。 + +为了在 Ubuntu/Debian 系统上编译 [taos-tools](https://github.com/taosdata/taos-tools) 需要安装如下软件： + +```bash +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config +``` + ### CentOS 7： ```bash -sudo yum install -y gcc gcc-c++ make cmake git +sudo yum install -y gcc gcc-c++ make cmake git openssl-devel ``` 安装 OpenJDK 8： @@ -74,10 +107,10 @@ sudo yum install -y java-1.8.0-openjdk sudo yum install -y maven ``` -### CentOS 8 & Fedora: +### CentOS 8 & Fedora ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git +sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel ``` 安装 OpenJDK 8： @@ -92,6 +125,33 @@ sudo dnf install -y java-1.8.0-openjdk sudo dnf install -y maven ``` +#### 在 CentOS 上构建 taosTools 安装依赖软件 + +为了在 CentOS 上构建 [taosTools](https://github.com/taosdata/taos-tools) 需要安装如下依赖软件 + +```bash +sudo yum install zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +``` + +注意：由于 snappy 缺乏 pkg-config 支持 +（参考 [链接](https://github.com/google/snappy/pull/86)），会导致 +cmake 提示无法发现 libsnappy，实际上工作正常。 + +### 设置 golang 开发环境 + +TDengine 包含数个使用 Go 语言开发的组件，请参考 golang.org 官方文档设置 go 开发环境。 + +请使用 1.14 及以上版本。对于中国用户，我们建议使用代理来加速软件包下载。 + +``` +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.cn,direct +``` + +### 设置 rust 开发环境 + +TDengine 包含数个使用 Rust 语言开发的组件. 请参考 rust-lang.org 官方文档设置 rust 开发环境。 + ## 获取源码 首先，你需要从 GitHub 克隆源码： @@ -107,22 +167,41 @@ Go 连接器和 Grafana 插件在其他独立仓库，如果安装它们的话 git submodule update --init --recursive ``` +如果使用 https 协议下载比较慢，可以通过修改 ~/.gitconfig 文件添加以下两行设置使用 ssh 协议下载。需要首先上传 ssh 密钥到 GitHub，详细方法请参考 GitHub 官方文档。 + +``` +[url "git@github.com:"] + insteadOf = https://github.com/ +``` + ## 构建 TDengine ### Linux 系统 +可以运行代码仓库中的 `build.sh` 脚本编译出 TDengine 和 taosTools（包含 taosBenchmark 和 taosdump）。 + ```bash -mkdir debug && cd debug -cmake .. && cmake --build . +./build.sh ``` -您可以选择使用 Jemalloc 作为内存分配器，替代默认的 glibc： +这个脚本等价于执行如下命令： + +```bash +git submodule update --init --recursive +mkdir debug +cd debug +cmake .. -DBUILD_TOOLS=true +make +``` + +您也可以选择使用 jemalloc 作为内存分配器，替代默认的 glibc： + ```bash apt install autoconf cmake .. -DJEMALLOC_ENABLED=true ``` -在X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 +在 X86-64、X86、arm64、arm32 和 mips64 平台上，TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型，如 aarch64 或 aarch32 等。 aarch64： @@ -157,7 +236,7 @@ nmake 如果你使用的是 Visual Studio 2019 或 2017 版本： -打开cmd.exe，执行 vcvarsall.bat 时，为 64 位操作系统指定“x64”，为 32 位操作系统指定“x86”。 +打开 cmd.exe，执行 vcvarsall.bat 时，为 64 位操作系统指定“x64”，为 32 位操作系统指定“x86”。 ```bash mkdir debug && cd debug @@ -174,9 +253,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -如果你使用的是 Visual Studio 2022 版本， 脚本 `vcvarsall.bat` 的默认安装路径是 `C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat`。 - -### Mac OS X 系统 +### macOS 系统 安装 Xcode 命令行工具和 cmake. 在 Catalina 和 Big Sur 操作系统上，需要安装 XCode 11.4+ 版本。 @@ -187,13 +264,17 @@ cmake .. && cmake --build . # 安装 -生成完成后，安装 TDengine（下文给出的指令以 Linux 为例，如果是在 Windows 下，那么对应的指令会是 `nmake install`）： +## Linux 系统 + +生成完成后，安装 TDengine： ```bash sudo make install ``` 用户可以在[文件目录结构](https://www.taosdata.com/cn/documentation/administrator#directories)中了解更多在操作系统中生成的目录或文件。 +从 2.0 版本开始, 从源代码安装也会为 TDengine 配置服务管理。 +用户也可以选择[从安装包中安装](https://www.taosdata.com/en/getting-started/#Install-from-Package)。 安装成功后，在终端中启动 TDengine 服务： @@ -209,6 +290,40 @@ taos 如果 TDengine Shell 连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印出错误消息。 +## Windows 系统 + +生成完成后，安装 TDengine： + +```cmd +nmake install +``` + +## macOS 系统 + +生成完成后，安装 TDengine： + +```bash +sudo make install +``` + +安装成功后，如果想以服务形式启动，先配置 `.plist` 文件，在终端中执行： + +```bash +sudo cp ../packaging/macOS/com.taosdata.tdengine.plist /Library/LaunchDaemons +``` + +在终端中启动 TDengine 服务： + +```bash +sudo launchctl load /Library/LaunchDaemons/com.taosdata.tdengine.plist +``` + +在终端中停止 TDengine 服务： + +```bash +sudo launchctl unload /Library/LaunchDaemons/com.taosdata.tdengine.plist +``` + ## 快速运行 如果不希望以服务方式运行 TDengine，也可以在终端中直接运行它。也即在生成完成后，执行以下命令（在 Windows 下，生成的可执行文件会带有 .exe 后缀，例如会名为 taosd.exe ）： @@ -227,15 +342,15 @@ taos # 体验 TDengine -在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。 +在 TDengine 终端中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行插入查询操作。 -```bash -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; +```sql +CREATE DATABASE demo; +USE demo; +CREATE TABLE t (ts TIMESTAMP, speed INT); +INSERT INTO t VALUES('2019-07-15 00:00:00', 10); +INSERT INTO t VALUES('2019-07-15 01:00:00', 20); +SELECT * FROM t; ts | speed | =================================== 19-07-15 00:00:00.000| 10| @@ -247,33 +362,35 @@ Query OK, 2 row(s) in set (0.001700s) ## 官方连接器 -TDengine 提供了丰富的应用程序开发接口，其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用： +TDengine 提供了丰富的应用程序开发接口，其中包括 C/C++、Java、Python、Go、Node.js、C# 、RESTful 等，便于用户快速开发应用： -- Java +- [Java](https://www.taosdata.com/cn/documentation/connector/java) -- C/C++ +- [C/C++](https://www.taosdata.com/cn/documentation/connector#c-cpp) -- Python +- [Python](https://www.taosdata.com/cn/documentation/connector#python) -- Go +- [Go](https://www.taosdata.com/cn/documentation/connector#go) -- RESTful API +- [RESTful API](https://www.taosdata.com/cn/documentation/connector#restful) -- Node.js +- [Node.js](https://www.taosdata.com/cn/documentation/connector#nodejs) + +- [Rust](https://www.taosdata.com/cn/documentation/connector/rust) ## 第三方连接器 TDengine 社区生态中也有一些非常友好的第三方连接器，可以通过以下链接访问它们的源码。 -- [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust) +- [Rust Bindings](https://github.com/songtianyi/tdengine-rust-bindings/tree/master/examples) - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos) -- [Lua Connector](https://github.com/taosdata/TDengine/tree/develop/tests/examples/lua) +- [Lua Connector](https://github.com/taosdata/TDengine/tree/develop/examples/lua) # 运行和添加测试例 TDengine 的测试框架和所有测试例全部开源。 -点击 [这里](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md)，了解如何运行测试例和添加新的测试例。 +点击 [这里](https://github.com/taosdata/TDengine/blob/develop/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md)，了解如何运行测试例和添加新的测试例。 # 成为社区贡献者 @@ -281,8 +398,8 @@ TDengine 的测试框架和所有测试例全部开源。 # 加入技术交流群 -TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine"，加小T为好友，即可入群。 +TDengine 官方社群「物联网大数据群」对外开放，欢迎您加入讨论。搜索微信号 "tdengine"，加小 T 为好友，即可入群。 -# [谁在使用TDengine](https://github.com/taosdata/TDengine/issues/2432) +# [谁在使用 TDengine](https://github.com/taosdata/TDengine/issues/2432) -欢迎所有 TDengine 用户及贡献者在 [这里](https://github.com/taosdata/TDengine/issues/2432) 分享您在当前工作中开发/使用 TDengine 的故事。 +欢迎所有 TDengine 用户及贡献者在 [这里](https://github.com/taosdata/TDengine/issues/2432) 分享您在当前工作中开发/使用 TDengine 的故事。 diff --git a/README.md b/README.md index 48349e891e..d4b8321813 100644 --- a/README.md +++ b/README.md @@ -1,116 +1,218 @@ +

+

+ + TDengine + +

+

+ [![Build Status](https://cloud.drone.io/api/badges/taosdata/TDengine/status.svg?ref=refs/heads/master)](https://cloud.drone.io/taosdata/TDengine) [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) [![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine) -[![TDengine](TDenginelogo.png)](https://www.taosdata.com) - -English | [简体中文](./README-CN.md) +English | [简体中文](README-CN.md) | We are hiring, check [here](https://tdengine.com/careers) # What is TDengine？ -TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html), designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and cost of development and operation. +TDengine is a high-performance, scalable time-series database with SQL support. Its code including cluster feature is open source under [GNU AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html). Besides the database, it provides caching, stream processing, data subscription and other functionalities to reduce the complexity and cost of development and operation. TDengine differentiates itself from other TSDBs with the following advantages. -- **10x Faster on Insert/Query Speeds**: Through the innovative design on storage, on a single-core machine, over 20K requests can be processed, millions of data points can be ingested, and over 10 million data points can be retrieved in a second. It is 10 times faster than other databases. +- **High Performance**: TDengine outperforms other time series databases in data ingestion and querying while significantly reducing storage cost and compute costs, with an innovatively designed and purpose-built storage engine. -- **1/5 Hardware/Cloud Service Costs**: Compared with typical big data solutions, less than 1/5 of computing resources are required. Via column-based storage and tuned compression algorithms for different data types, less than 1/10 of storage space is needed. +- **Scalable**: TDengine provides out-of-box scalability and high-availability through its native distributed design. Nodes can be added through simple configuration to achieve greater data processing power. In addition, this feature is open source. -- **Full Stack for Time-Series Data**: By integrating a database with message queuing, caching, and stream computing features together, it is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software. It makes the system architecture much simpler and more robust. +- **SQL Support**: TDengine uses SQL as the query language, thereby reducing learning and migration costs, while adding SQL extensions to handle time-series data better, and supporting convenient and flexible schemaless data ingestion. -- **Powerful Data Analysis**: Whether it is 10 years or one minute ago, data can be queried just by specifying the time range. Data can be aggregated over time, multiple time streams or both. Ad Hoc queries or analyses can be executed via TDengine shell, Python, R or Matlab. +- **All in One**: TDengine has built-in caching, stream processing and data subscription functions, it is no longer necessary to integrate Kafka/Redis/HBase/Spark or other software in some scenarios. It makes the system architecture much simpler and easy to maintain. -- **Seamless Integration with Other Tools**: Telegraf, Grafana, Matlab, R, and other tools can be integrated with TDengine without a line of code. MQTT, OPC, Hadoop, Spark, and many others will be integrated soon. +- **Seamless Integration**: Without a single line of code, TDengine provide seamless integration with third-party tools such as Telegraf, Grafana, EMQX, Prometheus, StatsD, collectd, etc. More will be integrated. -- **Zero Management, No Learning Curve**: It takes only seconds to download, install, and run it successfully; there are no other dependencies. Automatic partitioning on tables or DBs. Standard SQL is used, with C/C++, Python, JDBC, Go and RESTful connectors. +- **Zero Management**: Installation and cluster setup can be done in seconds. Data partitioning and sharding are executed automatically. TDengine’s running status can be monitored via Grafana or other DevOps tools. + +- **Zero Learning Cost**: With SQL as the query language, support for ubiquitous tools like Python, Java, C/C++, Go, Rust, Node.js connectors, there is zero learning cost. + +- **Interactive Console**: TDengine provides convenient console access to the database to run ad hoc queries, maintain the database, or manage the cluster without any programming. + +TDengine can be widely applied to Internet of Things (IoT), Connected Vehicles, Industrial IoT, DevOps, energy, finance and many other scenarios. # Documentation + For user manual, system design and architecture, engineering blogs, refer to [TDengine Documentation](https://www.taosdata.com/en/documentation/)(中文版请点击[这里](https://www.taosdata.com/cn/documentation20/)) - for details. The documentation from our website can also be downloaded locally from *documentation/tdenginedocs-en* or *documentation/tdenginedocs-cn*. +for details. The documentation from our website can also be downloaded locally from _documentation/tdenginedocs-en_ or _documentation/tdenginedocs-cn_. # Building -At the moment, TDengine only supports building and running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or from the source code. This quick guide is for installation from the source only. -To build TDengine, use [CMake](https://cmake.org/) 2.8.12.x or higher versions in the project directory. +At the moment, TDengine server only supports running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or build it from the source code. This quick guide is for installation from the source only. -## Install tools +To build TDengine, use [CMake](https://cmake.org/) 3.0.2 or higher versions in the project directory. + +## Install build dependencies + +### Ubuntu 16.04 and above or Debian -### Ubuntu 16.04 and above & Debian: ```bash -sudo apt-get install -y gcc cmake build-essential git +sudo apt-get install -y gcc cmake build-essential git libssl-dev ``` -### Ubuntu 14.04: +### Ubuntu 14.04 + ```bash sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 export PATH=/usr/lib/binutils-2.26/bin:$PATH ``` -To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. +To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. + To install openjdk-8: + ```bash sudo apt-get install -y openjdk-8-jdk ``` To install Apache Maven: + ```bash -sudo apt-get install -y maven +sudo apt-get install -y maven ``` -### Centos 7: +#### Install build dependencies for taosTools + +We provide a few useful tools such as taosBenchmark (was named taosdemo) and taosdump. They were part of TDengine. From TDengine 2.4.0.0, taosBenchmark and taosdump were not released together with TDengine. +By default, TDengine compiling does not include taosTools. You can use 'cmake .. -DBUILD_TOOLS=true' to make them be compiled with TDengine. + +To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed. + ```bash -sudo yum install -y gcc gcc-c++ make cmake git +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config +``` + +### CentOS 7 + +```bash +sudo yum install epel-release +sudo yum update +sudo yum install -y gcc gcc-c++ make cmake3 git openssl-devel +sudo ln -sf /usr/bin/cmake3 /usr/bin/cmake ``` To install openjdk-8: + ```bash sudo yum install -y java-1.8.0-openjdk ``` To install Apache Maven: + ```bash sudo yum install -y maven ``` -### Centos 8 & Fedora: +### CentOS 8 & Fedora + ```bash -sudo dnf install -y gcc gcc-c++ make cmake epel-release git +sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel ``` To install openjdk-8: + ```bash sudo dnf install -y java-1.8.0-openjdk ``` To install Apache Maven: + ```bash sudo dnf install -y maven ``` +#### Install build dependencies for taosTools on CentOS + +To build the [taosTools](https://github.com/taosdata/taos-tools) on CentOS, the following packages need to be installed. + +```bash +sudo yum install zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +``` + +Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it lead a cmake prompt libsnappy not found. But snappy will works well. + +### Setup golang environment + +TDengine includes few components developed by Go language. Please refer to golang.org official documentation for golang environment setup. + +Please use version 1.14+. For the user in China, we recommend using a proxy to accelerate package downloading. + +``` +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.cn,direct +``` + +### Setup rust environment + +TDengine includees few compoments developed by Rust language. Please refer to rust-lang.org official documentation for rust environment setup. + ## Get the source codes First of all, you may clone the source codes from github: + ```bash git clone https://github.com/taosdata/TDengine.git cd TDengine ``` -The connectors for go & grafana have been moved to separated repositories, +The connectors for go & Grafana and some tools have been moved to separated repositories, so you should run this command in the TDengine directory to install them: + ```bash git submodule update --init --recursive ``` +You can modify the file ~/.gitconfig to use ssh protocol instead of https for better download speed. You need to upload ssh public key to GitHub first. Please refer to GitHub official documentation for detail. + +``` +[url "git@github.com:"] + insteadOf = https://github.com/ +``` + ## Build TDengine ### On Linux platform +You can run the bash script `build.sh` to build both TDengine and taosTools including taosBenchmark and taosdump as below: + ```bash -mkdir debug && cd debug -cmake .. && cmake --build . +./build.sh +``` + +It equals to execute following commands: + +```bash +git submodule update --init --recursive +mkdir debug +cd debug +cmake .. -DBUILD_TOOLS=true +make +``` + +Note TDengine 2.3.x.0 and later use a component named 'taosAdapter' to play http daemon role by default instead of the http daemon embedded in the early version of TDengine. The taosAdapter is programmed by go language. If you pull TDengine source code to the latest from an existing codebase, please execute 'git submodule update --init --recursive' to pull taosAdapter source code. Please install go language version 1.14 or above for compiling taosAdapter. If you meet difficulties regarding 'go mod', especially you are from China, you can use a proxy to solve the problem. + +``` +go env -w GO111MODULE=on +go env -w GOPROXY=https://goproxy.cn,direct +``` + +The embedded http daemon still be built from TDengine source code by default. Or you can use the following command to choose to build taosAdapter. + +``` +cmake .. -DBUILD_HTTP=false ``` You can use Jemalloc as memory allocator instead of glibc: + ``` apt install autoconf cmake .. -DJEMALLOC_ENABLED=true @@ -120,24 +222,28 @@ TDengine build script can detect the host machine's architecture on X86-64, X86, You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: + ```bash cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` aarch32: + ```bash cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` mips64: + ```bash cmake .. -DCPUTYPE=mips64 && cmake --build . ``` ### On Windows platform -If you use Visual Studio 2013, please open a command window by executing "cmd.exe". +If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. + ```cmd mkdir debug && cd debug "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < amd64 | x86 > @@ -145,7 +251,7 @@ cmake .. -G "NMake Makefiles" nmake ``` -If you use Visual Studio 2019 or 2017: +If you use the Visual Studio 2019 or 2017: please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. @@ -158,15 +264,14 @@ nmake ``` Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: + ```cmd mkdir debug && cd debug cmake .. -G "NMake Makefiles" nmake ``` -If you use Visual Studio 2022, the only change is the default path of `vcvarsall.bat`, which is `C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat`. - -### On Mac OS X platform +### On macOS platform Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. @@ -177,7 +282,10 @@ cmake .. && cmake --build . # Installing -After building successfully, TDengine can be installed by: (On Windows platform, the following command should be `nmake install`) +## On Linux platform + +After building successfully, TDengine can be installed by + ```bash sudo make install ``` @@ -186,68 +294,129 @@ Users can find more information about directories installed on the system in the Users can also choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) for it. To start the service after installation, in a terminal, use: + ```bash sudo systemctl start taosd ``` Then users can use the [TDengine shell](https://www.taosdata.com/en/getting-started/#TDengine-Shell) to connect the TDengine server. In a terminal, use: + ```bash taos ``` If TDengine shell connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. +### Install TDengine by apt-get + +If you use Debian or Ubuntu system, you can use 'apt-get' command to install TDengine from official repository. Please use following commands to setup: + +``` +wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - +echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list +[Optional] echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list +sudo apt-get update +apt-cache policy tdengine +sudo apt-get install tdengine +``` + +## On Windows platform + +After building successfully, TDengine can be installed by: + +```cmd +nmake install +``` + +## On macOS platform + +After building successfully, TDengine can be installed by: + +```bash +sudo make install +``` + +To start the service after installation, config `.plist` file first, in a terminal, use: + +```bash +sudo cp ../packaging/macOS/com.taosdata.tdengine.plist /Library/LaunchDaemons +``` + +To start the service, in a terminal, use: + +```bash +sudo launchctl load /Library/LaunchDaemons/com.taosdata.tdengine.plist +``` + +To stop the service, in a terminal, use: + +```bash +sudo launchctl unload /Library/LaunchDaemons/com.taosdata.tdengine.plist +``` + ## Quick Run If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: (We take Linux as an example, command on Windows will be `taosd.exe`) + ```bash ./build/bin/taosd -c test/cfg ``` In another terminal, use the TDengine shell to connect the server: + ```bash ./build/bin/taos -c test/cfg ``` -option "-c test/cfg" specifies the system configuration file directory. +option "-c test/cfg" specifies the system configuration file directory. # Try TDengine + It is easy to run SQL commands from TDengine shell which is the same as other SQL databases. + ```sql -create database db; -use db; -create table t (ts timestamp, a int); -insert into t values ('2019-07-15 00:00:00', 1); -insert into t values ('2019-07-15 01:00:00', 2); -select * from t; -drop database db; +CREATE DATABASE demo; +USE demo; +CREATE TABLE t (ts TIMESTAMP, speed INT); +INSERT INTO t VALUES('2019-07-15 00:00:00', 10); +INSERT INTO t VALUES('2019-07-15 01:00:00', 20); +SELECT * FROM t; + ts | speed | +=================================== + 19-07-15 00:00:00.000| 10| + 19-07-15 01:00:00.000| 20| +Query OK, 2 row(s) in set (0.001700s) ``` # Developing with TDengine -### Official Connectors + +## Official Connectors TDengine provides abundant developing tools for users to develop on TDengine. Follow the links below to find your desired connectors and relevant documentation. -- [Java](https://www.taosdata.com/en/documentation/connector/#Java-Connector) -- [C/C++](https://www.taosdata.com/en/documentation/connector/#C/C++-Connector) -- [Python](https://www.taosdata.com/en/documentation/connector/#Python-Connector) -- [Go](https://www.taosdata.com/en/documentation/connector/#Go-Connector) -- [RESTful API](https://www.taosdata.com/en/documentation/connector/#RESTful-Connector) -- [Node.js](https://www.taosdata.com/en/documentation/connector/#Node.js-Connector) +- [Java](https://www.taosdata.com/en/documentation/connector/java) +- [C/C++](https://www.taosdata.com/en/documentation/connector#c-cpp) +- [Python](https://www.taosdata.com/en/documentation/connector#python) +- [Go](https://www.taosdata.com/en/documentation/connector#go) +- [RESTful API](https://www.taosdata.com/en/documentation/connector#restful) +- [Node.js](https://www.taosdata.com/en/documentation/connector#nodejs) +- [Rust](https://www.taosdata.com/en/documentation/connector/rust) -### Third Party Connectors +## Third Party Connectors The TDengine community has also kindly built some of their own connectors! Follow the links below to find the source code for them. -- [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust) +- [Rust Bindings](https://github.com/songtianyi/tdengine-rust-bindings/tree/master/examples) - [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos) - [Lua Connector](https://github.com/taosdata/TDengine/tree/develop/tests/examples/lua) -# How to run the test cases and how to add a new test case? - TDengine's test framework and all test cases are fully open source. - Please refer to [this document](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md) for how to run test and develop new test case. +# How to run the test cases and how to add a new test case + +TDengine's test framework and all test cases are fully open source. +Please refer to [this document](https://github.com/taosdata/TDengine/blob/develop/tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md) for how to run test and develop new test case. # TDengine Roadmap + - Support event-driven stream computing - Support user defined functions - Support MQTT connection diff --git a/cmake/cmake.define b/cmake/cmake.define index 1cc28ec6fd..a33db902ca 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -18,6 +18,14 @@ if (NOT DEFINED TD_GRANT) SET(TD_GRANT FALSE) endif() +IF ("${WEBSOCKET}" MATCHES "true") + SET(TD_WEBSOCKET TRUE) + MESSAGE("Enable websocket") + ADD_DEFINITIONS(-DWEBSOCKET) +ELSE () + SET(TD_WEBSOCKET FALSE) +ENDIF () + IF ("${BUILD_HTTP}" STREQUAL "") IF (TD_LINUX) IF (TD_ARM_32) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 74efb4c026..5d7f1bbe70 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -28,15 +28,23 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - void* meta; - int32_t metaLen; - tmq_get_raw_meta(msg, &meta, &metaLen); + tmq_raw_data *raw = tmq_get_raw_meta(msg); + if(raw){ + TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", "abc1", 0); + if (pConn == NULL) { + return; + } + int32_t ret = taos_write_raw_meta(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + free(raw); + taos_close(pConn); + } char* result = tmq_get_json_meta(msg); if(result){ printf("meta result: %s\n", result); free(result); } - printf("meta, len is %d\n", metaLen); + printf("meta:%p\n", raw); return; } while (1) { diff --git a/include/client/taos.h b/include/client/taos.h index 216a5832b0..362782b420 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -260,15 +260,16 @@ enum tmq_res_t { }; typedef enum tmq_res_t tmq_res_t; +typedef struct tmq_raw_data tmq_raw_data; -DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len); -DLL_EXPORT int32_t taos_write_raw_meta(TAOS *res, void *raw_meta, int32_t raw_meta_len); -DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed. -DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); -DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); -DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); +DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); +DLL_EXPORT tmq_raw_data *tmq_get_raw_meta(TAOS_RES *res); +DLL_EXPORT int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta); +DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); // Returning null means error. Returned result need to be freed. +DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); /* ------------------------------ TMQ END -------------------------------- */ diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 779453f714..28d771bbbd 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -55,7 +55,8 @@ enum { enum { STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_BLOCK, - STREAM_INPUT__DATA_SCAN, + STREAM_INPUT__TABLE_SCAN, + STREAM_INPUT__TQ_SCAN, STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__TRIGGER, STREAM_INPUT__CHECKPOINT, @@ -122,7 +123,8 @@ enum { }; typedef struct { - int8_t fetchType; + int8_t fetchType; + STqOffsetVal offset; union { SSDataBlock data; void* meta; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 8b64287033..af333f72aa 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -231,7 +231,7 @@ SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); -SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); +SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 20f9503150..23fd858e32 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -446,11 +446,16 @@ typedef struct { int32_t ast1Len; int32_t ast2Len; SArray* pColumns; // array of SField + int32_t cVersion; SArray* pTags; // array of SField + int32_t tVersion; SArray* pFuncs; char* pComment; char* pAst1; char* pAst2; + tb_uid_t suid; + int8_t source; // 1-taosX or 0-taosClient + int8_t reserved[8]; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -458,8 +463,11 @@ int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pR void tFreeSMCreateStbReq(SMCreateStbReq* pReq); typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; - int8_t igNotExists; + char name[TSDB_TABLE_FNAME_LEN]; + int8_t igNotExists; + tb_uid_t suid; + int8_t source; // 1-taosX or 0-taosClient + int8_t reserved[8]; } SMDropStbReq; int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); diff --git a/include/common/tname.h b/include/common/tname.h index 3bf1cee870..77965947ad 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -37,6 +37,8 @@ typedef struct SName { char tname[TSDB_TABLE_NAME_LEN]; } SName; +SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName); + int32_t tNameExtractFullName(const SName* name, char* dst); int32_t tNameLen(const SName* name); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index f512587880..1edf41b6f7 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -174,7 +174,13 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le */ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); + +int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset); + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); + +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo); void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index a569c8de54..4d27325d75 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,7 +172,13 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); +typedef enum { + SHOULD_FREE_COLDATA = 0x1, // the newly created column data needs to be destroyed. + DELEGATED_MGMT_COLDATA = 0x2, // input column data should not be released. +} ECOLDATA_MGMT_TYPE_E; + struct SScalarParam { + ECOLDATA_MGMT_TYPE_E type; SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index c3007306ae..a4aec72d4f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -96,6 +96,9 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols char* tableName, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); +int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); +SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); +SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); #ifdef __cplusplus } #endif diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 43792b5415..6d9f29906b 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -24,42 +24,14 @@ extern "C" { #endif -#define wFatal(...) \ - { \ - if (wDebugFlag & DEBUG_FATAL) { \ - taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \ - } \ - } -#define wError(...) \ - { \ - if (wDebugFlag & DEBUG_ERROR) { \ - taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ - } \ - } -#define wWarn(...) \ - { \ - if (wDebugFlag & DEBUG_WARN) { \ - taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); \ - } \ - } -#define wInfo(...) \ - { \ - if (wDebugFlag & DEBUG_INFO) { \ - taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); \ - } \ - } -#define wDebug(...) \ - { \ - if (wDebugFlag & DEBUG_DEBUG) { \ - taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); \ - } \ - } -#define wTrace(...) \ - { \ - if (wDebugFlag & DEBUG_TRACE) { \ - taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); \ - } \ - } +// clang-format off +#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", DEBUG_DEBUG, wDebugFlag, __VA_ARGS__); }} +#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }} +// clang-format on #define WAL_PROTO_VER 0 #define WAL_NOSUFFIX_LEN 20 @@ -194,6 +166,7 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver); SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); int32_t walReadVer(SWalReader *pRead, int64_t ver); +int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); // only for tq usage diff --git a/include/util/tencode.h b/include/util/tencode.h index a13afd4448..e318d4f240 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -477,22 +477,6 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { return n; \ } while (0) -#define tGetV(p, v) \ - do { \ - int32_t n = 0; \ - if (v) *v = 0; \ - for (;;) { \ - if (p[n] <= 0x7f) { \ - if (v) (*v) |= (p[n] << (7 * n)); \ - n++; \ - break; \ - } \ - if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ - n++; \ - } \ - return n; \ - } while (0) - // PUT static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { if (p) ((uint8_t*)p)[0] = v; @@ -607,7 +591,22 @@ static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { return sizeof(int64_t); } -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { + int32_t n = 0; + + if (v) *v = 0; + for (;;) { + if (p[n] <= 0x7f) { + if (v) (*v) |= (((uint16_t)p[n]) << (7 * n)); + n++; + break; + } + if (v) (*v) |= (((uint16_t)(p[n] & 0x7f)) << (7 * n)); + n++; + } + + return n; +} static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { int32_t n; @@ -619,7 +618,22 @@ static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { return n; } -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { + int32_t n = 0; + + if (v) *v = 0; + for (;;) { + if (p[n] <= 0x7f) { + if (v) (*v) |= (((uint32_t)p[n]) << (7 * n)); + n++; + break; + } + if (v) (*v) |= (((uint32_t)(p[n] & 0x7f)) << (7 * n)); + n++; + } + + return n; +} static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { int32_t n; @@ -631,7 +645,22 @@ static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { return n; } -static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { + int32_t n = 0; + + if (v) *v = 0; + for (;;) { + if (p[n] <= 0x7f) { + if (v) (*v) |= (((uint64_t)p[n]) << (7 * n)); + n++; + break; + } + if (v) (*v) |= (((uint64_t)(p[n] & 0x7f)) << (7 * n)); + n++; + } + + return n; +} static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { int32_t n; diff --git a/include/util/tlog.h b/include/util/tlog.h index a519aaa9b7..a8c9eeabde 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -94,7 +94,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } // clang-format on - +#define BUF_PAGE_DEBUG #ifdef __cplusplus } #endif diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 5217a82295..c9957fc89c 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -37,4 +37,5 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then fi # there can not libtaos.so*, otherwise ln -s error -${csudo}rm -f ${install_main_dir}/driver/libtaos* || : +${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : +${csudo}rm -f ${install_main_dir}/driver/libtaosws.* || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 501398f350..5ff970bb70 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -29,8 +29,12 @@ else ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : + ${csudo}rm -f ${inc_link_dir}/taosdef.h || : + ${csudo}rm -f ${inc_link_dir}/taoserror.h || : ${csudo}rm -f ${inc_link_dir}/taosudf.h || : + ${csudo}rm -f ${inc_link_dir}/taosws.h || : ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : ${csudo}rm -f ${log_link_dir} || : ${csudo}rm -f ${data_link_dir} || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 043c1456b8..1a4131ec6f 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -30,6 +30,7 @@ mkdir -p ${pkg_dir} cd ${pkg_dir} libfile="libtaos.so.${tdengine_ver}" +wslibfile="libtaosws.so" # create install dir install_home_path="/usr/local/taos" @@ -67,10 +68,12 @@ fi cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver +cp ${compile_dir}/build/lib/${wslibfile} ${pkg_dir}${install_home_path}/driver ||: cp ${compile_dir}/../include/client/taos.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/common/taosdef.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/util/taoserror.h ${pkg_dir}${install_home_path}/include cp ${compile_dir}/../include/libs/function/taosudf.h ${pkg_dir}${install_home_path}/include +cp ${compile_dir}/../src/inc/taosws.h ${pkg_dir}${install_home_path}/include ||: cp -r ${top_dir}/examples/* ${pkg_dir}${install_home_path}/examples #cp -r ${top_dir}/src/connector/python ${pkg_dir}${install_home_path}/connector #cp -r ${top_dir}/src/connector/go ${pkg_dir}${install_home_path}/connector diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index f440f72aa2..02e7e9b5e5 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -42,6 +42,7 @@ echo version: %{_version} echo buildroot: %{buildroot} libfile="libtaos.so.%{_version}" +wslibfile="libtaosws.so" # create install path, and cp file mkdir -p %{buildroot}%{homepath}/bin @@ -74,10 +75,12 @@ if [ -f %{_compiledir}/build/bin/taosadapter ]; then cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver +cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: cp %{_compiledir}/../include/client/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/common/taosdef.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/util/taoserror.h %{buildroot}%{homepath}/include cp %{_compiledir}/../include/libs/function/taosudf.h %{buildroot}%{homepath}/include +cp %{_compiledir}/../src/inc/taosws.h %{buildroot}%{homepath}/include ||: #cp -r %{_compiledir}/../src/connector/python %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/go %{buildroot}%{homepath}/connector #cp -r %{_compiledir}/../src/connector/nodejs %{buildroot}%{homepath}/connector diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 230172cdc6..77817e5cf6 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -229,9 +229,13 @@ function install_lib() { ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -s ${lib_link_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so || : + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : + + ${csudo}ln -s ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi ${csudo}ldconfig @@ -315,11 +319,16 @@ function install_jemalloc() { function install_header() { ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : + + ${csudo}rm -f ${inc_link_dir}/taosws.h || : + ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + + ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : } function add_newHostname_to_hosts() { diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 4e4ebb72c0..8ad42811d4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -294,21 +294,29 @@ function install_avro() { function install_lib() { # Remove links ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : fi if [ "$osType" != "Darwin" ]; then ${csudo}cp ${binary_dir}/build/lib/libtaos.so.${verNumber} \ ${install_main_dir}/driver && - ${csudo}chmod 777 ${install_main_dir}/driver/* + ${csudo}chmod 777 ${install_main_dir}/driver/libtaos.so.${verNumber} + + ${csudo}cp ${binary_dir}/build/lib/libtaosws.so \ + ${install_main_dir}/driver && + ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.so ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.so ${lib_link_dir}/libtaosws.so || : if [ -d "${lib64_link_dir}" ]; then ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 ${csudo}ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so + ${csudo}ln -sf ${lib64_link_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : fi else ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ @@ -337,8 +345,8 @@ function install_lib() { fi install_jemalloc - install_avro lib - install_avro lib64 + #install_avro lib + #install_avro lib64 if [ "$osType" != "Darwin" ]; then ${csudo}ldconfig @@ -350,11 +358,19 @@ function install_header() { if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ + ${csudo}rm -f ${inc_link_dir}/taosws.h || : + + ${csudo}cp -f ${source_dir}/src/inc/taos.h ${source_dir}/src/inc/taosdef.h ${source_dir}/src/inc/taoserror.h \ ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* + + ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + + ${csudo}ln -s ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h || : else ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ ${install_main_dir}/include || diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e098acdb40..7edab2141b 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -93,8 +93,11 @@ else fi lib_files="${build_dir}/lib/libtaos.so.${version}" +wslib_files="${build_dir}/lib/libtaosws.so." header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" +wsheader_files="${code_dir}/inc/taosws.h" + if [ "$dbName" != "taos" ]; then cfg_dir="${top_dir}/../enterprise/packaging/cfg" else @@ -110,6 +113,9 @@ init_file_rpm=${script_dir}/../rpm/taosd # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc + +${wsheader_files} ${install_dir}/inc || : + mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/${configFile} ${install_dir}/cfg/${configFile} if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then @@ -284,6 +290,7 @@ fi # Copy driver mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt +cp ${wslib_files} ${install_dir}/driver || : # Copy connector if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index d34c81a4f5..ec836f2eac 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -102,7 +102,10 @@ function clean_local_bin() { function clean_lib() { # Remove link ${csudo}rm -f ${lib_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib_link_dir}/libtaosws.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : + ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : } @@ -111,6 +114,8 @@ function clean_header() { ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : ${csudo}rm -f ${inc_link_dir}/taoserror.h || : + + ${csudo}rm -f ${inc_link_dir}/taosws.h || : } function clean_config() { diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 5039f8bbaf..b6972e4670 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1570,41 +1570,40 @@ static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsV return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + *tsVal = timeDouble; size_t typeLen = strlen(type->valuestring); if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { // seconds - timeDouble = timeDouble * 1e9; + *tsVal = *tsVal * NANOSECOND_PER_SEC; + timeDouble = timeDouble * NANOSECOND_PER_SEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - *tsVal = timeDouble; } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { switch (type->valuestring[0]) { case 'm': case 'M': // milliseconds - timeDouble = timeDouble * 1e6; + *tsVal = *tsVal * NANOSECOND_PER_MSEC; + timeDouble = timeDouble * NANOSECOND_PER_MSEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - *tsVal = timeDouble; break; case 'u': case 'U': // microseconds - timeDouble = timeDouble * 1e3; + *tsVal = *tsVal * NANOSECOND_PER_USEC; + timeDouble = timeDouble * NANOSECOND_PER_USEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - *tsVal = timeDouble; break; case 'n': case 'N': - // nanoseconds - *tsVal = timeDouble; break; default: return TSDB_CODE_TSC_INVALID_JSON; @@ -1641,21 +1640,23 @@ static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { if (timeDouble < 0) { return TSDB_CODE_TSC_INVALID_TIME_STAMP; } + uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); + tsVal = (int64_t)timeDouble; if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - timeDouble = timeDouble * 1e9; + tsVal = tsVal * NANOSECOND_PER_SEC; + timeDouble = timeDouble * NANOSECOND_PER_SEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - tsVal = timeDouble; } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeDouble = timeDouble * 1e6; + tsVal = tsVal * NANOSECOND_PER_MSEC; + timeDouble = timeDouble * NANOSECOND_PER_MSEC; if (smlDoubleToInt64OverFlow(timeDouble)) { smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); return TSDB_CODE_TSC_INVALID_TIME_STAMP; } - tsVal = timeDouble; } else if (timeDouble == 0) { tsVal = taosGetTimestampNs(); } else { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 2132f7a2d6..6710f73fc6 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -106,6 +106,12 @@ struct tmq_t { tsem_t rspSem; }; +struct tmq_raw_data{ + void *raw_meta; + int32_t raw_meta_len; + int16_t raw_meta_type; +}; + enum { TMQ_VG_STATUS__IDLE = 0, TMQ_VG_STATUS__WAIT, @@ -396,7 +402,7 @@ int32_t tmqCommitCb2(void* param, SDataBuf* pBuf, int32_t code) { } #endif - /*tscDebug("receive offset commit cb of %s on vg %d, offset is %ld", pParam->pOffset->subKey, pParam->->vgId, + /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, * pOffset->version);*/ // count down waiting rsp @@ -471,8 +477,8 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT .handle = NULL, }; - tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, - pOffset->val.version); + tscDebug("consumer:%" PRId64 ", commit offset of %s on vgId:%d, offset is %" PRId64, tmq->consumerId, pOffset->subKey, + pVg->vgId, pOffset->val.version); // TODO: put into cb pVg->committedOffsetNew = pVg->currentOffsetNew; @@ -585,13 +591,14 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - tscDebug("consumer %ld begin commit for topic %s, vgNum %d", tmq->consumerId, pTopic->topicName, + tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgNum %d", tmq->consumerId, pTopic->topicName, (int32_t)taosArrayGetSize(pTopic->vgs)); for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId); + tscDebug("consumer:%" PRId64 ", begin commit for topic %s, vgId:%d", tmq->consumerId, pTopic->topicName, + pVg->vgId); if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { @@ -1082,13 +1089,13 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t epoch = pParam->epoch; taosMemoryFree(pParam); if (code != 0) { - tscWarn("msg discard from vg %d, epoch %d, code:%x", vgId, epoch, code); + tscWarn("msg discard from vgId:%d, epoch %d, code:%x", vgId, epoch, code); if (pMsg->pData) taosMemoryFree(pMsg->pData); if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { taosMemoryFree(pMsg->pData); - tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch); + tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; @@ -1104,7 +1111,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqEpoch = atomic_load_32(&tmq->epoch); if (msgEpoch < tmqEpoch) { // do not write into queue since updating epoch reset - tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch, + tscWarn("msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); tsem_post(&tmq->rspSem); taosMemoryFree(pMsg->pData); @@ -1112,7 +1119,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } if (msgEpoch != tmqEpoch) { - tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); + tscWarn("mismatch rsp from vgId:%d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); } // handle meta rsp @@ -1121,7 +1128,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { taosMemoryFree(pMsg->pData); - tscWarn("msg discard from vg %d, epoch %d since out of memory", vgId, epoch); + tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; } @@ -1143,8 +1150,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); - tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId, - pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType); + tscDebug("consumer:%" PRId64 ", recv poll: vgId:%d, req offset %" PRId64 ", rsp offset %" PRId64 " type %d", + tmq->consumerId, pVg->vgId, pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, + rspType); taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1163,7 +1171,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, + tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); @@ -1182,14 +1190,14 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); if (pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); + tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); char buf[50]; tFormatOffset(buf, 50, &pVgCur->currentOffsetNew); - tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, - buf); + tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, + vgKey, buf); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); } } @@ -1202,7 +1210,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { topic.topicName = strdup(pTopicEp->topic); tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); + tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); int32_t vgNumGet = taosArrayGetSize(pTopicEp->vgs); topic.vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); @@ -1215,7 +1223,8 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { offsetNew = *pOffset; } - /*tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,*/ + /*tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64 ", vgKey is %s", + * tmq->consumerId, epoch,*/ /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, @@ -1249,7 +1258,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumGet = taosArrayGetSize(pRsp->topics); char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer %ld update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, + tscDebug("consumer:%" PRId64 ", update ep epoch %d to epoch %d, topic num: %d", tmq->consumerId, tmq->epoch, epoch, topicNumGet); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { @@ -1270,19 +1279,19 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { topic.topicName = strdup(pTopicEp->topic); tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); - tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); + tscDebug("consumer:%" PRId64 ", update topic: %s", tmq->consumerId, topic.topicName); int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { // find old topic SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); if (pTopicCur->vgs && strcmp(pTopicCur->topicName, pTopicEp->topic) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer %ld new vg num: %d", tmq->consumerId, vgNumCur); + tscDebug("consumer:%" PRId64 ", new vg num: %d", tmq->consumerId, vgNumCur); if (vgNumCur == 0) break; for (int32_t k = 0; k < vgNumCur; k++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, k); sprintf(vgKey, "%s:%d", topic.topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); + tscDebug("consumer:%" PRId64 ", epoch %d vgId:%d build %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey); taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); } break; @@ -1296,13 +1305,13 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("consumer %ld(epoch %d) original offset of vg %d is %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer:%" PRId64 ", (epoch %d) original offset of vgId:%d is %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("consumer %ld(epoch %d) receive offset of vg %d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, + tscDebug("consumer:%" PRId64 ", (epoch %d) receive offset of vgId:%d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); } - tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64, tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1336,7 +1345,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { int8_t async = pParam->async; pParam->code = code; if (code != 0) { - tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); + tscError("consumer:%" PRId64 ", get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async); goto END; } @@ -1345,7 +1354,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { // Epoch will only increase when received newer epoch ep msg SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); - tscDebug("consumer %ld recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); + tscDebug("consumer:%" PRId64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); if (head->epoch <= epoch) { goto END; } @@ -1353,8 +1362,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { if (!async) { SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); - /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ - /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + /*printf("rsp epoch %" PRId64 " sz %" PRId64 "\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %" PRId64 " sz %" PRId64 "\n", tmq->epoch, tmq->clientTopics->size);*/ tmqUpdateEp2(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { @@ -1389,7 +1398,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); if (epStatus == 1) { int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); - tscTrace("consumer %ld skip ask ep cnt %d", tmq->consumerId, epSkipCnt); + tscTrace("consumer:%" PRId64 ", skip ask ep cnt %d", tmq->consumerId, epSkipCnt); if (epSkipCnt < 5000) return 0; } atomic_store_32(&tmq->epSkipCnt, 0); @@ -1439,7 +1448,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer %ld ask ep", tmq->consumerId); + tscDebug("consumer:%" PRId64 ", ask ep", tmq->consumerId); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -1555,14 +1564,14 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus != TMQ_VG_STATUS__IDLE) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscTrace("consumer %ld epoch %d skip vg %d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); + tscTrace("consumer:%" PRId64 ", epoch %d skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; /*if (vgSkipCnt < 10000) continue;*/ #if 0 if (skipCnt < 30000) { continue; } else { - tscDebug("consumer %ld skip vg %d skip too much reset", tmq->consumerId, pVg->vgId); + tscDebug("consumer:%" PRId64 ",skip vgId:%d skip too much reset", tmq->consumerId, pVg->vgId); } #endif } @@ -1611,9 +1620,9 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { char offsetFormatBuf[80]; tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew); - tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId, + tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); - /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ + /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; tmq->pollCnt++; @@ -1660,7 +1669,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ + /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { @@ -1682,7 +1691,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { int32_t consumerEpoch = atomic_load_32(&tmq->epoch); if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ + /*printf("vgId:%d offset %" PRId64 " up to %" PRId64 "\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffsetNew.version = pollRspWrapper->metaRsp.rspOffset; pVg->currentOffsetNew.type = TMQ_OFFSET__LOG; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); @@ -1701,7 +1710,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmqHandleNoPollRsp(tmq, rspWrapper, &reset); taosFreeQitem(rspWrapper); if (pollIfReset && reset) { - tscDebug("consumer %ld reset and repoll", tmq->consumerId); + tscDebug("consumer:%" PRId64 ", reset and repoll", tmq->consumerId); tmqPollImpl(tmq, timeout); } } @@ -1741,7 +1750,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { int64_t endTime = taosGetTimestampMs(); int64_t leftTime = endTime - startTime; if (leftTime > timeout) { - tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); + tscDebug("consumer:%" PRId64 ", (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); return NULL; } tsem_timewait(&tmq->rspSem, leftTime * 1000); @@ -1839,14 +1848,16 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } -int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) { +tmq_raw_data *tmq_get_raw_meta(TAOS_RES* res) { if (TD_RES_TMQ_META(res)) { + tmq_raw_data *raw = taosMemoryCalloc(1, sizeof(tmq_raw_data)); SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - *raw_meta = pMetaRspObj->metaRsp.metaRsp; - *raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; - return 0; + raw->raw_meta = pMetaRspObj->metaRsp.metaRsp; + raw->raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; + raw->raw_meta_type = pMetaRspObj->metaRsp.resMsgType; + return raw; } - return -1; + return NULL; } static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ @@ -2249,6 +2260,515 @@ char *tmq_get_json_meta(TAOS_RES *res){ return NULL; } +static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ + SVCreateStbReq req = {0}; + SDecoder coder; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pTscObj) { + code = TSDB_CODE_TSC_DISCONNECTED; + goto end; + } + + code = buildRequest(pTscObj, "", 0, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if(!pRequest->pDb){ + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVCreateStbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + // build create stable + pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); + for(int32_t i = 0; i < req.schemaRow.nCols; i++){ + SSchema* pSchema = req.schemaRow.pSchema + i; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + strcpy(field.name, pSchema->name); + taosArrayPush(pReq.pColumns, &field); + } + pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); + for(int32_t i = 0; i < req.schemaTag.nCols; i++){ + SSchema* pSchema = req.schemaTag.pSchema + i; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + strcpy(field.name, pSchema->name); + taosArrayPush(pReq.pTags, &field); + } + pReq.cVersion = req.schemaRow.version; + pReq.tVersion = req.schemaTag.version; + pReq.numOfColumns = req.schemaRow.nCols; + pReq.numOfTags = req.schemaTag.nCols; + pReq.commentLen = -1; + pReq.suid = req.suid; + pReq.source = 1; + + SName tableName; + tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); + + SCmdMsgInfo pCmdMsg = {0}; + pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery = {0}; + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; + + launchQueryImpl(pRequest, &pQuery, true, NULL); + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); + + end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + tDecoderClear(&coder); + return code; +} + +static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ + SVDropStbReq req = {0}; + SDecoder coder; + SMDropStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pTscObj) { + code = TSDB_CODE_TSC_DISCONNECTED; + goto end; + } + + code = buildRequest(pTscObj, "", 0, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if(!pRequest->pDb){ + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVDropStbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + // build drop stable + pReq.igNotExists = true; + pReq.source = 1; + pReq.suid = req.suid; + SName tableName; + tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); + + SCmdMsgInfo pCmdMsg = {0}; + pCmdMsg.epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_DROP_STB; + pCmdMsg.msgLen = tSerializeSMDropStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); + + SQuery pQuery = {0}; + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; + + launchQueryImpl(pRequest, &pQuery, true, NULL); + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); + + end: + destroyRequest(pRequest); + tDecoderClear(&coder); + return code; +} + +typedef struct SVgroupCreateTableBatch { + SVCreateTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupCreateTableBatch; + +static void destroyCreateTbReqBatch(void* data) { + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; + taosArrayDestroy(pTbBatch->req.pArray); +} + +static int32_t taosCreateTable(TAOS *taos, void *meta, int32_t metaLen){ + SVCreateTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj *pRequest = NULL; + SQuery *pQuery = NULL; + SHashObj *pVgroupHashmap = NULL; + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + + if (NULL == pTscObj) { + code = TSDB_CODE_TSC_DISCONNECTED; + goto end; + } + + code = buildRequest(pTscObj, "", 0, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if(!pRequest->pDb){ + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + SVCreateTbReq *pCreateReq = NULL; + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + // loop to create table + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pCreateReq = req.pReqs + iReq; + + SVgroupInfo pInfo = {0}; + SName pName; + toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); + if (pTableBatch == NULL) { + SVgroupCreateTableBatch tBatch = {0}; + tBatch.info = pInfo; + strcpy(tBatch.dbName, pRequest->pDb); + + tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq)); + taosArrayPush(tBatch.req.pArray, pCreateReq); + + taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, pCreateReq); + } + } + + SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); + if (NULL == pBufArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_CREATE_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, false, NULL); + pQuery = NULL; // no need to free in the end + code = pRequest->code; + + end: + taosHashCleanup(pVgroupHashmap); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +typedef struct SVgroupDropTableBatch { + SVDropTbBatchReq req; + SVgroupInfo info; + char dbName[TSDB_DB_NAME_LEN]; +} SVgroupDropTableBatch; + +static void destroyDropTbReqBatch(void* data) { + SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; + taosArrayDestroy(pTbBatch->req.pArray); +} + +static int32_t taosDropTable(TAOS *taos, void *meta, int32_t metaLen){ + SVDropTbBatchReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj *pRequest = NULL; + SQuery *pQuery = NULL; + SHashObj *pVgroupHashmap = NULL; + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + + if (NULL == pTscObj) { + code = TSDB_CODE_TSC_DISCONNECTED; + goto end; + } + + code = buildRequest(pTscObj, "", 0, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if(!pRequest->pDb){ + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + SVDropTbReq *pDropReq = NULL; + SCatalog *pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pVgroupHashmap) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + // loop to create table + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + pDropReq = req.pReqs + iReq; + + SVgroupInfo pInfo = {0}; + SName pName; + toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); + if (pTableBatch == NULL) { + SVgroupDropTableBatch tBatch = {0}; + tBatch.info = pInfo; + tBatch.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq)); + taosArrayPush(tBatch.req.pArray, pDropReq); + + taosHashPut(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId), &tBatch, sizeof(tBatch)); + } else { // add to the correct vgroup + taosArrayPush(pTableBatch->req.pArray, pDropReq); + } + } + + SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); + if (NULL == pBufArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_DROP_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, false, NULL); + pQuery = NULL; // no need to free in the end + code = pRequest->code; + + end: + taosHashCleanup(pVgroupHashmap); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj *pRequest = NULL; + SQuery *pQuery = NULL; + SArray *pArray = NULL; + SVgDataBlocks *pVgData = NULL; + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + + if (NULL == pTscObj) { + code = TSDB_CODE_TSC_DISCONNECTED; + goto end; + } + + code = buildRequest(pTscObj, "", 0, &pRequest); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + if(!pRequest->pDb){ + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; + } + // decode and process req + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + int32_t len = metaLen - sizeof(SMsgHead); + tDecoderInit(&coder, data, len); + if (tDecodeSVAlterTbReq(&coder, &req) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto end; + } + + // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS + if(req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS){ + goto end; + } + + SCatalog *pCatalog = NULL; + code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + + SVgroupInfo pInfo = {0}; + SName pName = {0}; + toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pArray = taosArrayInit(1, sizeof(void*)); + if (NULL == pArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + pVgData = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == pVgData) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgData->vg = pInfo; + pVgData->pData = taosMemoryMalloc(metaLen); + if (NULL == pVgData->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + memcpy(pVgData->pData, meta, metaLen); + ((SMsgHead*)pVgData->pData)->vgId = htonl(pInfo.vgId); + pVgData->size = metaLen; + pVgData->numOfTables = 1; + taosArrayPush(pArray, &pVgData); + + pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->msgType = TDMT_VND_ALTER_TABLE; + pQuery->stableQuery = false; + pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + + code = rewriteToVnodeModifyOpStmt(pQuery, pArray); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + launchQueryImpl(pRequest, pQuery, false, NULL); + pQuery = NULL; // no need to free in the end + pVgData = NULL; + pArray = NULL; + code = pRequest->code; + +end: + taosArrayDestroy(pArray); + if(pVgData) taosMemoryFreeClear(pVgData->pData); + taosMemoryFreeClear(pVgData); + destroyRequest(pRequest); + tDecoderClear(&coder); + qDestroyQuery(pQuery); + return code; +} + +int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data *raw_meta){ + if (!taos || !raw_meta) { + return TSDB_CODE_INVALID_PARA; + } + + if(raw_meta->raw_meta_type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_STB){ + return taosCreateStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_STB){ + return taosDropStb(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + }else if(raw_meta->raw_meta_type == TDMT_VND_CREATE_TABLE){ + return taosCreateTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + }else if(raw_meta->raw_meta_type == TDMT_VND_ALTER_TABLE){ + return taosAlterTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + }else if(raw_meta->raw_meta_type == TDMT_VND_DROP_TABLE){ + return taosDropTable(taos, raw_meta->raw_meta, raw_meta->raw_meta_len); + } + return TSDB_CODE_INVALID_PARA; +} + void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { tmqCommitInner2(tmq, msg, 0, 1, cb, param); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 832564e0db..a6062efb98 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1284,4 +1284,210 @@ TEST(testCase, sml_dup_time_Test) { ASSERT_EQ(taos_errno(pRes), 0); taos_free_result(pRes); } -*/ + + +TEST(testCase, sml_16960_Test) { +TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); +ASSERT_NE(taos, nullptr); + +TAOS_RES* pRes = taos_query(taos, "create database if not exists d16368 schemaless 1"); +taos_free_result(pRes); + +pRes = taos_query(taos, "use d16368"); +taos_free_result(pRes); + +const char *sql[] = { + "[\n" + "{\n" + "\"timestamp\":\n" + "\n" + "{ \"value\": 1349020800000, \"type\": \"ms\" }\n" + ",\n" + "\"value\":\n" + "\n" + "{ \"value\": 830525384, \"type\": \"int\" }\n" + ",\n" + "\"tags\": {\n" + "\"id\": \"stb00_0\",\n" + "\"t0\":\n" + "\n" + "{ \"value\": 83972721, \"type\": \"int\" }\n" + ",\n" + "\"t1\":\n" + "\n" + "{ \"value\": 539147525, \"type\": \"int\" }\n" + ",\n" + "\"t2\":\n" + "\n" + "{ \"value\": 618258572, \"type\": \"int\" }\n" + ",\n" + "\"t3\":\n" + "\n" + "{ \"value\": -10536201, \"type\": \"int\" }\n" + ",\n" + "\"t4\":\n" + "\n" + "{ \"value\": 349227409, \"type\": \"int\" }\n" + ",\n" + "\"t5\":\n" + "\n" + "{ \"value\": 249347042, \"type\": \"int\" }\n" + "},\n" + "\"metric\": \"stb0\"\n" + "},\n" + "{\n" + "\"timestamp\":\n" + "\n" + "{ \"value\": 1349020800001, \"type\": \"ms\" }\n" + ",\n" + "\"value\":\n" + "\n" + "{ \"value\": -588348364, \"type\": \"int\" }\n" + ",\n" + "\"tags\": {\n" + "\"id\": \"stb00_0\",\n" + "\"t0\":\n" + "\n" + "{ \"value\": 83972721, \"type\": \"int\" }\n" + ",\n" + "\"t1\":\n" + "\n" + "{ \"value\": 539147525, \"type\": \"int\" }\n" + ",\n" + "\"t2\":\n" + "\n" + "{ \"value\": 618258572, \"type\": \"int\" }\n" + ",\n" + "\"t3\":\n" + "\n" + "{ \"value\": -10536201, \"type\": \"int\" }\n" + ",\n" + "\"t4\":\n" + "\n" + "{ \"value\": 349227409, \"type\": \"int\" }\n" + ",\n" + "\"t5\":\n" + "\n" + "{ \"value\": 249347042, \"type\": \"int\" }\n" + "},\n" + "\"metric\": \"stb0\"\n" + "},\n" + "{\n" + "\"timestamp\":\n" + "\n" + "{ \"value\": 1349020800002, \"type\": \"ms\" }\n" + ",\n" + "\"value\":\n" + "\n" + "{ \"value\": -370310823, \"type\": \"int\" }\n" + ",\n" + "\"tags\": {\n" + "\"id\": \"stb00_0\",\n" + "\"t0\":\n" + "\n" + "{ \"value\": 83972721, \"type\": \"int\" }\n" + ",\n" + "\"t1\":\n" + "\n" + "{ \"value\": 539147525, \"type\": \"int\" }\n" + ",\n" + "\"t2\":\n" + "\n" + "{ \"value\": 618258572, \"type\": \"int\" }\n" + ",\n" + "\"t3\":\n" + "\n" + "{ \"value\": -10536201, \"type\": \"int\" }\n" + ",\n" + "\"t4\":\n" + "\n" + "{ \"value\": 349227409, \"type\": \"int\" }\n" + ",\n" + "\"t5\":\n" + "\n" + "{ \"value\": 249347042, \"type\": \"int\" }\n" + "},\n" + "\"metric\": \"stb0\"\n" + "},\n" + "{\n" + "\"timestamp\":\n" + "\n" + "{ \"value\": 1349020800003, \"type\": \"ms\" }\n" + ",\n" + "\"value\":\n" + "\n" + "{ \"value\": -811250191, \"type\": \"int\" }\n" + ",\n" + "\"tags\": {\n" + "\"id\": \"stb00_0\",\n" + "\"t0\":\n" + "\n" + "{ \"value\": 83972721, \"type\": \"int\" }\n" + ",\n" + "\"t1\":\n" + "\n" + "{ \"value\": 539147525, \"type\": \"int\" }\n" + ",\n" + "\"t2\":\n" + "\n" + "{ \"value\": 618258572, \"type\": \"int\" }\n" + ",\n" + "\"t3\":\n" + "\n" + "{ \"value\": -10536201, \"type\": \"int\" }\n" + ",\n" + "\"t4\":\n" + "\n" + "{ \"value\": 349227409, \"type\": \"int\" }\n" + ",\n" + "\"t5\":\n" + "\n" + "{ \"value\": 249347042, \"type\": \"int\" }\n" + "},\n" + "\"metric\": \"stb0\"\n" + "},\n" + "{\n" + "\"timestamp\":\n" + "\n" + "{ \"value\": 1349020800004, \"type\": \"ms\" }\n" + ",\n" + "\"value\":\n" + "\n" + "{ \"value\": -330340558, \"type\": \"int\" }\n" + ",\n" + "\"tags\": {\n" + "\"id\": \"stb00_0\",\n" + "\"t0\":\n" + "\n" + "{ \"value\": 83972721, \"type\": \"int\" }\n" + ",\n" + "\"t1\":\n" + "\n" + "{ \"value\": 539147525, \"type\": \"int\" }\n" + ",\n" + "\"t2\":\n" + "\n" + "{ \"value\": 618258572, \"type\": \"int\" }\n" + ",\n" + "\"t3\":\n" + "\n" + "{ \"value\": -10536201, \"type\": \"int\" }\n" + ",\n" + "\"t4\":\n" + "\n" + "{ \"value\": 349227409, \"type\": \"int\" }\n" + ",\n" + "\"t5\":\n" + "\n" + "{ \"value\": 249347042, \"type\": \"int\" }\n" + "},\n" + "\"metric\": \"stb0\"\n" + "}\n" + "]" +}; + +pRes = taos_schemaless_insert(taos, (char**)sql, sizeof(sql)/sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); +ASSERT_EQ(taos_errno(pRes), 0); +taos_free_result(pRes); +} +*/ \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 8b15f4a101..4e17a9aea0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1356,7 +1356,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) return col; } -SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { +SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) { ASSERT(pBlock != NULL); if (index >= taosArrayGetSize(pBlock->pDataBlock)) { return NULL; @@ -1735,9 +1735,9 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag, + len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id:%" PRIu64 "|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); - if (len >= size -1) return dumpBuf; + if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { len += snprintf(dumpBuf + len, size - len, "%s |", flag); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 192a41a70e..9972b2aa08 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -516,6 +516,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->cVersion) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfTags; ++i) { SField *pField = taosArrayGet(pReq->pTags, i); if (tEncodeI8(&encoder, pField->type) < 0) return -1; @@ -524,6 +526,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->tVersion) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { const char *pFunc = taosArrayGet(pReq->pFuncs, i); if (tEncodeCStr(&encoder, pFunc) < 0) return -1; @@ -538,6 +542,11 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (pReq->ast2Len > 0) { if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1; } + if (tEncodeI64(&encoder, pReq->suid) < 0) return -1; + if (tEncodeI8(&encoder, pReq->source) < 0) return -1; + for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) { + if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1; + } tEndEncode(&encoder); @@ -585,6 +594,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } } + if (tDecodeI32(&decoder, &pReq->cVersion) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfTags; ++i) { SField field = {0}; if (tDecodeI8(&decoder, &field.type) < 0) return -1; @@ -597,6 +608,8 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR } } + if (tDecodeI32(&decoder, &pReq->tVersion) < 0) return -1; + for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { char pFunc[TSDB_FUNC_NAME_LEN] = {0}; if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1; @@ -624,6 +637,12 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1; } + if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->source) < 0) return -1; + for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) { + if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -645,6 +664,11 @@ int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + if (tEncodeI64(&encoder, pReq->suid) < 0) return -1; + if (tEncodeI8(&encoder, pReq->source) < 0) return -1; + for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) { + if (tEncodeI8(&encoder, pReq->reserved[i]) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -659,6 +683,12 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->suid) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->source) < 0) return -1; + for (int32_t i = 0; i < sizeof(pReq->reserved)/sizeof(int8_t); ++i) { + if (tDecodeI8(&decoder, &pReq->reserved[i]) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -5406,11 +5436,11 @@ int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { snprintf(buf, maxLen, "offset(reset to latest)"); } else if (pVal->type == TMQ_OFFSET__LOG) { - snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version); + snprintf(buf, maxLen, "offset(log) ver:%" PRId64, pVal->version); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { - snprintf(buf, maxLen, "offset(ss data) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + snprintf(buf, maxLen, "offset(ss data) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts); } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) { - snprintf(buf, maxLen, "offset(ss meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + snprintf(buf, maxLen, "offset(ss meta) uid:%" PRId64 ", ts:%" PRId64, pVal->uid, pVal->ts); } else { ASSERT(0); } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fb3a2d7464..7183153824 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -115,6 +115,14 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in #endif +SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { + pName->type = TSDB_TABLE_NAME_T; + pName->acctId = acctId; + strcpy(pName->dbname, pDbName); + strcpy(pName->tname, pTableName); + return pName; +} + int32_t tNameExtractFullName(const SName* name, char* dst) { assert(name != NULL && dst != NULL); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 59b442881a..5f982ad3a4 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -21,7 +21,7 @@ extern SConfig *tsCfg; static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set local info, dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosThreadRwlockWrlock(&pMgmt->pData->lock); pMgmt->pData->dnodeId = pCfg->dnodeId; pMgmt->pData->clusterId = pCfg->clusterId; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 7f3f76b4b6..a171d2e1e4 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -167,7 +167,7 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { if (pMsg == NULL) return -1; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 993a604b51..5ffddd0127 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -384,6 +384,8 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index ab9d3f67e7..ecbb695e02 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -277,7 +277,7 @@ int32_t dmRunDnode(SDnode *pDnode) { while (1) { if (pDnode->stop) { - dInfo("dnode is about to stop"); + dInfo("TDengine is about to stop"); dmSetStatus(pDnode, DND_STAT_STOPPED); dmStopNodes(pDnode); dmCloseNodes(pDnode); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 14867ff693..b94c60c4ab 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -34,8 +34,6 @@ extern "C" { #endif // clang-format off - - #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} #define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} @@ -49,7 +47,6 @@ extern "C" { #define mGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);} #define mGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);} #define mGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);} - // clang-format on #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5b5de10fba..5e7c19ce6a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -92,7 +92,7 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); ASSERT(pConsumer); - mInfo("receive consumer lost msg, consumer id %ld, status %s", pLostMsg->consumerId, + mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId, mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { @@ -124,7 +124,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); ASSERT(pConsumer); - mInfo("receive consumer recover msg, consumer id %ld, status %s", pRecoverMsg->consumerId, + mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { @@ -296,7 +296,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2. check epoch, only send ep info when epoches do not match if (epoch != serverEpoch) { taosRLockLatch(&pConsumer->lock); - mInfo("process ask ep, consumer %ld(epoch %d), server epoch %d", consumerId, epoch, serverEpoch); + mInfo("process ask ep, consumer:%" PRId64 "(epoch %d), server epoch %d", consumerId, epoch, serverEpoch); int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); @@ -441,7 +441,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SMqTopicObj topicObj = {0}; memcpy(&topicObj, pTopic, sizeof(SMqTopicObj)); topicObj.refConsumerCnt = pTopic->refConsumerCnt + 1; - mInfo("subscribe topic %s by consumer %ld cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup, + mInfo("subscribe topic %s by consumer:%" PRId64 ",cgroup %s, refcnt %d", pTopic->name, consumerId, cgroup, topicObj.refConsumerCnt); if (mndSetTopicCommitLogs(pMnode, pTrans, &topicObj) != 0) goto SUBSCRIBE_OVER; #endif @@ -451,7 +451,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerOld = mndAcquireConsumer(pMnode, consumerId); if (pConsumerOld == NULL) { - mInfo("receive subscribe request from new consumer: %ld", consumerId); + mInfo("receive subscribe request from new consumer:%" PRId64, consumerId); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); tstrncpy(pConsumerNew->clientId, subscribe.clientId, 256); @@ -472,7 +472,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumerOld->status); - mInfo("receive subscribe request from old consumer: %ld, current status: %s", consumerId, + mInfo("receive subscribe request from old consumer:%" PRId64 ", current status: %s", consumerId, mndConsumerStatusName(status)); if (status != MQ_CONSUMER_STATUS__READY) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6ead922d95..9ce108b789 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -406,7 +406,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } if (statusReq.dnodeId == 0) { - mInfo("dnode:%d, %s first access, set clusterId %" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); + mInfo("dnode:%d, %s first access, clusterId:%" PRId64, pDnode->id, pDnode->ep, pMnode->clusterId); } else { if (statusReq.clusterId != pMnode->clusterId) { if (pDnode != NULL) { diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index e2b20b2163..00753de0ec 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -185,7 +185,7 @@ static int32_t mndProcessCommitOffsetReq(SRpcMsg *pMsg) { for (int32_t i = 0; i < commitOffsetReq.num; i++) { SMqOffset *pOffset = &commitOffsetReq.offsets[i]; - mInfo("commit offset %ld to vg %d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId, + mInfo("commit offset %" PRId64 " to vgId:%d of consumer group %s on topic %s", pOffset->offset, pOffset->vgId, pOffset->cgroup, pOffset->topicName); if (mndMakePartitionKey(key, pOffset->cgroup, pOffset->topicName, pOffset->vgId) < 0) { mError("submit offset to topic %s failed", pOffset->topicName); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e084710e25..282fc44965 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -705,10 +705,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); pDst->createdTime = taosGetTimestampMs(); pDst->updateTime = pDst->createdTime; - pDst->uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + pDst->uid = (pCreate->source == 1) ? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); pDst->dbUid = pDb->uid; - pDst->tagVer = 1; - pDst->colVer = 1; + pDst->tagVer = (pCreate->source == 1) ? pCreate->tVersion : 1; + pDst->colVer = (pCreate->source == 1) ? pCreate->cVersion : 1; pDst->smaVer = 1; pDst->nextColId = 1; pDst->maxdelay[0] = pCreate->delay1; @@ -1752,6 +1752,11 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { } } + if (dropReq.source == 1 && pStb->uid != dropReq.suid){ + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + goto _OVER; + } + pDb = mndAcquireDbByStb(pMnode, dropReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8dde3e92d8..d67e4e8783 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -235,7 +235,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vg %d from consumer %ld", pVgEp->vgId, consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId); } taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); // put into removed @@ -255,7 +255,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vg %d from unassigned", pVgEp->vgId); + mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId); } } @@ -298,7 +298,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); } imbCnt++; } @@ -312,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR .pVgEp = pVgEp, }; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("mq rebalance: remove vg %d from consumer %ld (first scan)", pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId, pConsumerEp->consumerId); } } } @@ -329,7 +329,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp)); taosArrayPush(pOutput->newConsumers, &consumerId); - mInfo("mq rebalance: add new consumer %ld", consumerId); + mInfo("mq rebalance: add new consumer:%" PRId64, consumerId); } } @@ -354,7 +354,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } @@ -372,7 +372,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); - mInfo("mq rebalance: add vg %d to consumer %ld (second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("mq rebalance: add vgId:%d to consumer:%" PRId64 ",(second scan)", pRebVg->pVgEp->vgId, pConsumerEp->consumerId); } } else { // if all consumer is removed, put all vg into unassigned @@ -385,7 +385,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR ASSERT(pRebOutput->newConsumerId == -1); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->rebVgs, pRebOutput); - mInfo("mq rebalance: unassign vg %d (second scan)", pRebOutput->pVgEp->vgId); + mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId); } } @@ -393,7 +393,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR mInfo("rebalance calculation completed, rebalanced vg:"); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); - mInfo("vgId:%d, moved from consumer %" PRId64 " to consumer %" PRId64, pOutputRebVg->pVgEp->vgId, + mInfo("vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId); } @@ -546,7 +546,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - ASSERT(pTopic); + /*ASSERT(pTopic);*/ + if (pTopic == NULL) { + mError("rebalance %s failed since topic %s was dropped, abort", pRebInfo->key, topic); + continue; + } taosRLockLatch(&pTopic->lock); rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 8d1220d596..f2a037ab82 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -558,7 +558,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since subscribed by consumer %ld in consumer group %s", dropReq.name, + mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s", dropReq.name, pConsumer->consumerId, pConsumer->cgroup); return -1; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index b13e654caf..0a5fe1001c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -111,7 +111,7 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) { streamSetupTrigger(pTask); - qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId, + qInfo("deploy stream: stream id %" PRId64 " task id %d child id %d on snode", pTask->streamId, pTask->taskId, pTask->selfChildId); taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *)); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 897e4f26e2..d20d79ee47 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -26,7 +26,6 @@ target_sources( "src/meta/metaSnapshot.c" # sma - "src/sma/sma.c" "src/sma/smaEnv.c" "src/sma/smaUtil.c" "src/sma/smaOpen.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b87b9a0512..ff29305b74 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -174,6 +174,9 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqSeekVer(STqReader *pReader, int64_t ver); +int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); + int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 2d6edae0e7..7183c423fe 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -47,7 +47,8 @@ struct SSmaEnv { }; typedef struct { - int32_t smaRef; + int8_t inited; + int32_t rsetId; } SSmaMgmt; #define SMA_ENV_LOCK(env) ((env)->lock) @@ -95,6 +96,7 @@ enum { TASK_TRIGGER_STAT_CANCELLED = 4, TASK_TRIGGER_STAT_FINISHED = 5, }; + void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -104,6 +106,10 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); + +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); + int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); int32_t tdLockSma(SSma *pSma); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 9c3bd85c71..b2ea08d50d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -129,6 +129,7 @@ typedef struct { static STqMgmt tqMgmt = {0}; // tqRead +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ae659dc396..34fac045a7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -163,6 +163,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool const char* stbFullName, int32_t vgId); // sma +int32_t smaInit(); +void smaCleanUp(); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c deleted file mode 100644 index 12f93f9400..0000000000 --- a/source/dnode/vnode/src/sma/sma.c +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" - -// functions for external invocation - -// TODO: Who is responsible for resource allocate and release? -int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { - int32_t code = TSDB_CODE_SUCCESS; - - if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { - smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { - int32_t code = TSDB_CODE_SUCCESS; - - if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { - smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { - smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); - } - smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); - return code; -} - - -// functions for internal invocation - -#if 0 - -/** - * @brief TODO: Assume that the final generated result it less than 3M - * - * @param pReq - * @param pDataBlocks - * @param vgId - * @param suid // TODO: check with Liao whether suid response is reasonable - * - * TODO: colId should be set - */ -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, - tb_uid_t suid, const char* stbName, bool isCreateCtb) { - int32_t sz = taosArrayGetSize(pDataBlocks); - int32_t bufSize = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; ++i) { - SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); - bufSize += sizeof(SSubmitBlk); - } - - *pReq = taosMemoryCalloc(1, bufSize); - if (!(*pReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - void* pDataBuf = *pReq; - - SArray* pTagArray = NULL; - int32_t msgLen = sizeof(SSubmitReq); - int32_t numOfBlks = 0; - int32_t schemaLen = 0; - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - SDataBlockInfo* pDataBlkInfo = &pDataBlock->info; - int32_t colNum = pDataBlkInfo->numOfCols; - int32_t rows = pDataBlkInfo->rows; - int32_t rowSize = pDataBlkInfo->rowSize; - int64_t groupId = pDataBlkInfo->groupId; - - if (rb.nCols != colNum) { - tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); - } - - if(isCreateCtb) { - SMetaReader mr = {0}; - const char* ctbName = buildCtbNameByGroupId(stbName, pDataBlock->info.groupId); - if (metaGetTableEntryByName(&mr, ctbName) != 0) { - smaDebug("vgId:%d, no tsma ctb %s exists", vgId, ctbName); - } - SVCreateTbReq ctbReq = {0}; - ctbReq.name = ctbName; - ctbReq.type = TSDB_CHILD_TABLE; - ctbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = colNum + PRIMARYKEY_TIMESTAMP_COL_ID, - .type = TSDB_DATA_TYPE_BIGINT, - .i64 = groupId}; - STag* pTag = NULL; - if(!pTagArray) { - pTagArray = taosArrayInit(1, sizeof(STagVal)); - if (!pTagArray) goto _err; - } - taosArrayClear(pTagArray); - taosArrayPush(pTagArray, &tagVal); - tTagNew(pTagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&ctbReq); - goto _err; - } - ctbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &ctbReq, schemaLen, code); - - tdDestroySVCreateTbReq(&ctbReq); - if (code < 0) { - goto _err; - } - } - - - - SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); - pSubmitBlk->suid = suid; - pSubmitBlk->uid = groupId; - pSubmitBlk->numOfRows = rows; - - msgLen += sizeof(SSubmitBlk); - int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf - bool isStartKey = false; - int32_t offset = 0; - for (int32_t k = 0; k < colNum; ++k) { // iterate by column - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - STColumn* pCol = &pTSchema->columns[k]; - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, - offset, k); - - } else { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, - true, offset, k); - } - break; - case TSDB_DATA_TYPE_NCHAR: { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, - offset, k); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, - offset, k); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_MEDIUMBLOB: - uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - TASSERT(0); - break; - default: - if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - if (pCol->type == pColInfoData->info.type) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, - k); - } else { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, - k); - } - } else { - uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - } - break; - } - offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation - } - dataLen += TD_ROW_LEN(rb.pBuf); -#ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(rb.pBuf, pTSchema, __func__); -#endif - } - - ++numOfBlks; - - pSubmitBlk->dataLen = dataLen; - msgLen += pSubmitBlk->dataLen; - } - - (*pReq)->length = msgLen; - - (*pReq)->header.vgId = htonl(vgId); - (*pReq)->header.contLen = htonl(msgLen); - (*pReq)->length = (*pReq)->header.contLen; - (*pReq)->numOfBlocks = htonl(numOfBlks); - SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); - while (numOfBlks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->padding = htonl(blk->padding); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); - } - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(*pReq); - taosArrayDestroy(pTagArray); - - return TSDB_CODE_FAILED; -} -#endif diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 6e75176136..e241c14fc7 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -121,7 +121,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { // step 3: perform persist task for qTaskInfo tdRSmaPersistExecImpl(pRSmaStat); - smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); return TSDB_CODE_SUCCESS; } @@ -173,6 +173,7 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { } if ((pDir = taosOpenDir(dir)) == NULL) { + regfree(®ex); terrno = TAOS_SYSTEM_ERROR(errno); smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index c7b938f884..bb8dd48236 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -18,7 +18,7 @@ typedef struct SSmaStat SSmaStat; #define RSMA_TASK_INFO_HASH_SLOT 8 -#define SMA_MGMT_REF_NUM 1024 +#define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; @@ -30,7 +30,62 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaE static void *tdFreeTSmaStat(STSmaStat *pStat); static void tdDestroyRSmaStat(void *pRSmaStat); +/** + * @brief rsma init + * + * @return int32_t + */ // implementation +int32_t smaInit() { + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&smaMgmt.inited, 0, 2); + if (old != 2) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + if (old == 0) { + smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); + + if (smaMgmt.rsetId < 0) { + smaError("failed to init sma rset since %s", terrstr()); + atomic_store_8(&smaMgmt.inited, 0); + return TSDB_CODE_FAILED; + } + + smaInfo("sma rset is initialized, rsetId:%d", smaMgmt.rsetId); + atomic_store_8(&smaMgmt.inited, 1); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief rsma cleanup + * + */ +void smaCleanUp() { + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&smaMgmt.inited, 1, 2); + if (old != 2) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + if (old == 1) { + smaInfo("sma rset is cleaned up, resetId:%d", smaMgmt.rsetId); + taosCloseRef(smaMgmt.rsetId); + atomic_store_8(&smaMgmt.inited, 0); + } +} static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) { SSmaEnv *pEnv = NULL; @@ -135,17 +190,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); // init smaMgmt - smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); - if (smaMgmt.smaRef < 0) { - smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } + smaInit(); - int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat); + int64_t refId = taosAddRef(smaMgmt.rsetId, pRSmaStat); if (refId < 0) { - smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno)); + smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), + refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, + smaMgmt.rsetId, SMA_MGMT_REF_NUM); } pRSmaStat->refId = refId; @@ -275,8 +329,13 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat); - if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) { - smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr()); + if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { + smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma), + RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr()); + ASSERT(0); + } else { + smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma), + RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId); } } else { ASSERT(0); @@ -323,7 +382,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { } break; default: - TASSERT(0); + smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4c2e606e0e..c44a46ac5a 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,7 +19,8 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid SSmaMgmt smaMgmt = { - .smaRef = -1, + .inited = 0, + .rsetId = -1, }; #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" @@ -608,9 +609,11 @@ _err: static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; SSma *pSma = NULL; - SRSmaStat *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId); + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + if (!pStat) { - smaDebug("rsma fetch task not start since already destroyed"); + smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pItem->refId); return; } @@ -622,9 +625,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: case TASK_TRIGGER_STAT_FINISHED: { - taosReleaseRef(smaMgmt.smaRef, pItem->refId); - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled", - SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is %" PRIi8 + ", rsetId rsetId:%" PRIi64 " refId:%d", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); return; } default: @@ -665,7 +669,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } _end: - taosReleaseRef(smaMgmt.smaRef, pItem->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); } static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, @@ -1258,7 +1262,8 @@ _end: } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); + smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), smaMgmt.rsetId, pRSmaStat->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); taosThreadExit(NULL); return NULL; } @@ -1283,7 +1288,9 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); + smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d)", SMA_VID(pRSmaStat->pSma), smaMgmt.rsetId, + pRSmaStat->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); } taosThreadAttrDestroy(&thAttr); @@ -1297,8 +1304,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *rsmaStat = param; - SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId); - + SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.rsetId, rsmaStat->refId); + ASSERT(0); if (!pRSmaStat) { smaDebug("rsma persistence task not start since already destroyed"); return; @@ -1341,5 +1348,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); } break; } - taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId); + taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId); } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index ecd8db8ea9..6d71f3a250 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -20,6 +20,36 @@ #define SMA_STORAGE_MINUTES_DAY 1440 #define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file +// TODO: Who is responsible for resource allocate and release? +int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + + if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { + smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + // TODO: destroy SSDataBlocks(msg) + return code; +} + +int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + + if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { + smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + // TODO: destroy SSDataBlocks(msg) + return code; +} + +int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { + smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); + } + smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); + return code; +} + /** * @brief Judge the tsma file split days * diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 2ce6d23648..22d82e8df0 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -294,4 +294,23 @@ int32_t tdRemoveTFile(STFile *pTFile) { } // smaXXXUtil ================ +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { + void *pResult = taosAcquireRef(rsetId, refId); + if (!pResult) { + smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + } else { + smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + } + return pResult; +} + +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { + if (taosReleaseRef(rsetId, refId) < 0) { + smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + return TSDB_CODE_FAILED; + } + smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + + return TSDB_CODE_SUCCESS; +} // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e21b0fe9e8..52949838b9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -112,7 +112,7 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, }; tmsgSendRsp(&resp); - tqDebug("vg %d from consumer %ld (epoch %d) send rsp, res msg type %d, reqOffset: %ld, rspOffset: %ld", + tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64, TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset); return 0; @@ -162,7 +162,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con char buf2[80]; tFormatOffset(buf1, 80, &pRsp->reqOffset); tFormatOffset(buf2, 80, &pRsp->rspOffset); - tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s", + tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%s, rspOffset:%s", TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); return 0; @@ -179,10 +179,10 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { tDecoderClear(&decoder); if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); } else if (offset.val.type == TMQ_OFFSET__LOG) { - tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey, + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, TD_VID(pTq->pVnode), offset.val.version); } else { ASSERT(0); @@ -244,22 +244,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { STqOffsetVal fetchOffsetNew; // 1.find handle - char buf[80]; - tFormatOffset(buf, 80, &reqOffset); - tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), buf); - STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); /*ASSERT(pHandle);*/ if (pHandle == NULL) { - tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode), - pReq->subKey); + tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId, + TD_VID(pTq->pVnode), pReq->subKey); return -1; } // check rebalance if (pHandle->consumerId != consumerId) { - tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld", + tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 + ", in vgId:%d, subkey %s, handle consumer id %" PRId64, consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); return -1; } @@ -270,6 +266,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); } + char buf[80]; + tFormatOffset(buf, 80, &reqOffset); + tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId, + pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); + // 2.reset offset if needed if (reqOffset.type > 0) { fetchOffsetNew = reqOffset; @@ -279,7 +280,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffsetNew = pOffset->val; char formatBuf[80]; tFormatOffset(formatBuf, 80, &fetchOffsetNew); - tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf); + tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, offset reset to %s", consumerId, pHandle->subKey, formatBuf); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -294,9 +295,30 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal)); + tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey, + fetchOffsetNew.version); + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + dataRsp.rspOffset = fetchOffsetNew; + code = 0; + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + taosArrayDestroy(dataRsp.blockDataLen); + taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree); + + if (dataRsp.withSchema) { + taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } + + if (dataRsp.withTbName) { + taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree); + } + return code; } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId, - TD_VID(pTq->pVnode), pReq->subKey); + tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64 + " in vg %d, subkey %s, reset none failed", + pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; } @@ -307,7 +329,24 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); - if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + fetchOffsetNew.version++; + if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { + ASSERT(0); + code = -1; + goto OVER; + } + if (dataRsp.blockNum == 0) { + // TODO add to async task + /*dataRsp.rspOffset.version--;*/ + } + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + goto OVER; + } + + if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { int64_t fetchVer = fetchOffsetNew.version + 1; SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { @@ -319,8 +358,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { - tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + tqWarn("tmq poll: consumer %ld (epoch %d), subkey %s, vg %d offset %" PRId64 + ", found new consumer epoch %d, discard req epoch %d", + consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); break; } @@ -337,8 +377,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SWalCont* pHead = &pCkHead->head; - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchVer, pHead->msgType); + tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, + pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; @@ -363,7 +403,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else { ASSERT(pHandle->fetchMeta); ASSERT(IS_META_MSG(pHead->msgType)); - tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); + tqInfo("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; /*metaRsp.reqOffset = pReq->reqOffset.version;*/ /*metaRsp.rspOffset = fetchVer;*/ @@ -384,7 +424,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosMemoryFree(pCkHead); } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts); + tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { ASSERT(0); } @@ -482,10 +522,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->execHandle.execTb.suid = req.suid; SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); vnodeGetCtbIdList(pTq->pVnode, req.suid, tbUidList); - tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid); + tqDebug("vgId:%d, tq try get suid:%" PRId64, pTq->pVnode->config.vgId, req.suid); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - tqDebug("vg %d, idx %d, uid: %ld", TD_VID(pTq->pVnode), i, tbUid); + tqDebug("vgId:%d, idx %d, uid:%" PRId64, TD_VID(pTq->pVnode), i, tbUid); } for (int32_t i = 0; i < 5; i++) { tqReaderSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); @@ -572,7 +612,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { streamSetupTrigger(pTask); - tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode)); + tqInfo("deploy stream task id %d child id %d on vgId:%d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode)); taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 16822e6003..d381cfcdc7 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -46,7 +46,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { @@ -59,6 +59,53 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i return 0; } +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { + qTaskInfo_t task = pExec->execCol.task[0]; + + if (qStreamPrepareScan1(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + pRsp->rspOffset.version--; + return 0; + } + + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + + if (pDataBlock != NULL) { + tqAddBlockDataToRsp(pDataBlock, pRsp); + if (pRsp->withTbName) { + int64_t uid = pExec->pExecReader[0]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp); + } + pRsp->blockNum++; + continue; + } + + void* meta = qStreamExtractMetaMsg(task); + if (meta != NULL) { + // tq add meta to rsp + } + + if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { + ASSERT(0); + } + + if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { + ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version); + } + + ASSERT(pRsp->rspOffset.type != 0); + + break; + } + + return 0; +} + int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) { ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->execCol.task[workerId]; @@ -67,7 +114,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S /*ASSERT(0);*/ /*}*/ - if (qStreamPrepareScan(task, offset.uid, offset.ts) < 0) { + if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) { ASSERT(0); } @@ -93,7 +140,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S if (qGetStreamScanStatus(task, &uid, &ts) < 0) { ASSERT(0); } - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); #endif } pRsp->blockNum++; @@ -129,7 +176,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } pRsp->blockNum++; } @@ -146,7 +193,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; @@ -164,7 +211,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 8561314431..ec9674d637 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -139,7 +139,7 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { ASSERT(0); - tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen); + tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); taosHashCancelIterate(pStore->pHash, pIter); return -1; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index bd7cda4de3..e9e5f6cd8b 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -223,7 +223,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo)); taosWUnLockLatch(&pHandle->pushHandle.lock); - tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + tqDebug("vgId:%d offset %" PRId64 " from consumer:%" PRId64 ", (epoch %d) send rsp, block num: %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64, TD_VID(pTq->pVnode), fetchOffset, pHandle->pushHandle.consumerId, pHandle->pushHandle.epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 53ef17a6ba..35179b0234 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -15,11 +15,6 @@ #include "tq.h" -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) { - /*if ()*/ - return 0; -} - int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t code = 0; taosThreadMutexLock(&pHandle->pWalReader->mutex); @@ -27,7 +22,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea while (1) { if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) { - tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", pHandle->consumerId, + tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) vgId:%d offset %" PRId64 ", no more log to return", pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode), offset); *fetchOffset = offset - 1; code = -1; @@ -84,8 +79,10 @@ STqReader* tqOpenReader(SVnode* pVnode) { return NULL; } - // TODO open - /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/ + pReader->pWalReader = walOpenReader(pVnode->pWal, NULL); + if (pReader->pWalReader == NULL) { + return NULL; + } pReader->pVnodeMeta = pVnode->pMeta; pReader->pMsg = NULL; @@ -106,12 +103,19 @@ void tqCloseReader(STqReader* pReader) { taosMemoryFree(pReader); } +int32_t tqSeekVer(STqReader* pReader, int64_t ver) { + // + return walReadSeekVer(pReader->pWalReader, ver); +} + int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { bool fromProcessedMsg = pReader->pMsg != NULL; while (1) { if (!fromProcessedMsg) { if (walNextValidMsg(pReader->pWalReader) < 0) { + ret->offset.type = TMQ_OFFSET__LOG; + ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; return -1; } @@ -130,19 +134,25 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { + ASSERT(0); + continue; +#if 0 if (fromProcessedMsg) { ret->fetchType = FETCH_TYPE__NONE; return 0; } else { break; } +#endif } - ret->fetchType = FETCH_TYPE__DATA; return 0; } if (fromProcessedMsg) { + ret->offset.type = TMQ_OFFSET__LOG; + ret->offset.version = pReader->ver; + ASSERT(pReader->ver != -1); ret->fetchType = FETCH_TYPE__NONE; return 0; } @@ -179,9 +189,9 @@ bool tqNextDataBlock(STqReader* pReader) { return true; } void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); - /*tqDebug("search uid %ld", pHandle->msgIter.uid);*/ + /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/ if (ret != NULL) { - /*tqDebug("find uid %ld", pHandle->msgIter.uid);*/ + /*tqDebug("find uid %" PRId64, pHandle->msgIter.uid);*/ return true; } } @@ -212,7 +222,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchema) taosMemoryFree(pReader->pSchema); pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion); if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid: %ld (suid: %ld), version %d, possibly dropped table", + tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; @@ -222,7 +232,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, true); if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid: %ld, version %d, possibly dropped table", + tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->msgIter.uid, pReader->cachedSchemaVer); /*ASSERT(0);*/ terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ea1dfc057f..0c5f851d97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -155,7 +155,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST /* tsdbCacheInsertLastrow(pCache, uid, row, dup); */ } } - } else { + } /*else { if (dup) { cacheRow = tdRowDup(row); } else { @@ -168,7 +168,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST if (status != TAOS_LRU_STATUS_OK) { code = -1; } - } + }*/ return code; } @@ -992,7 +992,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = maxKey}); // if (taosArrayPush(pColArray, pColVal) == NULL) { - if (taosArrayPush(pColArray, &(SLastCol){.ts = TSKEY_MAX, .colVal = *pColVal}) == NULL) { + if (taosArrayPush(pColArray, &(SLastCol){.ts = maxKey, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -1127,7 +1127,7 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH //*ppRow = (STSRow *)taosLRUCacheValue(pCache, h); } else { STSRow *pRow = NULL; - bool dup = false; + bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pRow); // if table's empty or error, return code of -1 if (code < 0 || pRow == NULL) { @@ -1139,7 +1139,14 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUH return 0; } - tsdbCacheInsertLastrow(pCache, pTsdb, uid, pRow, dup); + _taos_lru_deleter_t deleter = deleteTableCacheLastrow; + LRUStatus status = + taosLRUCacheInsert(pCache, key, keyLen, pRow, TD_ROW_LEN(pRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + // tsdbCacheInsertLastrow(pCache, pTsdb, uid, pRow, dup); h = taosLRUCacheLookup(pCache, key, keyLen); //*ppRow = (STSRow *)taosLRUCacheValue(pCache, h); } @@ -1202,7 +1209,7 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand if (status != TAOS_LRU_STATUS_OK) { code = -1; } - /* tsdbCacheInsertLast(pCache, uid, pRow); */ + h = taosLRUCacheLookup(pCache, key, keyLen); //*ppRow = (STSRow *)taosLRUCacheValue(pCache, h); } @@ -1235,9 +1242,23 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { getTableCacheKey(uid, 1, key, &keyLen); h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { - // clear last cache anyway, no matter where eKey ends. - taosLRUCacheRelease(pCache, h, true); + SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); + bool invalidate = false; + int16_t nCol = taosArrayGetSize(pLast); + for (int16_t iCol = 0; iCol < nCol; ++iCol) { + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); + if (eKey >= tTsVal->ts) { + invalidate = true; + break; + } + } + + if (invalidate) { + taosLRUCacheRelease(pCache, h, true); + } else { + taosLRUCacheRelease(pCache, h, false); + } // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 838c61109e..9e01468fc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2182,12 +2182,21 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret return VND_TSDB(pVnode); } -static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { + int64_t startVer = (pCond->startVersion == -1)? 0:pCond->startVersion; + if (VND_IS_RSMA(pVnode)) { - return (SVersionRange){.minVer = pCond->startVersion, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)}; + return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)}; } - return (SVersionRange){.minVer = pCond->startVersion, .maxVer = pVnode->state.applied}; + int64_t endVer = 0; + if (pCond->endVersion == -1) { // user not specified end version, set current maximum version of vnode as the endVersion + endVer = pVnode->state.applied; + } else { + endVer = (pCond->endVersion > pVnode->state.applied)? pVnode->state.applied:pCond->endVersion; + } + + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } // // todo not unref yet, since it is not support multi-group interpolation query diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index a331f88e61..e96ee03b03 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -424,10 +424,12 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx, uint8_t **ppBuf) { ASSERT(n == size - sizeof(TSCKSUM)); + tFree(pBuf); return code; _err: tsdbError("vgId:%d read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tFree(pBuf); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index ed8bdc17a5..2e628edb7a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1016,7 +1016,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS SColData *pColData; SColVal *pColVal; - ASSERT(nColData > 0); + if (nColData == 0) goto _exit; tRowIterInit(pIter, pRow, pTSchema); pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); @@ -1046,6 +1046,7 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS } } +_exit: pBlockData->nRow++; return code; diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index d0aede145e..69f87a2c1a 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -100,6 +100,7 @@ void vnodeCleanup() { walCleanUp(); tqCleanUp(); + smaCleanUp(); } int vnodeScheduleTask(int (*execute)(void*), void* arg) { diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f75ccba4bb..a16ba8b89e 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -338,12 +338,12 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); ASSERT(pSyncMsg != NULL); - code = syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg); + code = syncNodeOnRequestVoteSnapshotCb(pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); ASSERT(pSyncMsg != NULL); - code = syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg); + code = syncNodeOnRequestVoteReplySnapshotCb(pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_BATCH) { SyncAppendEntriesBatch *pSyncMsg = syncAppendEntriesBatchFromRpcMsg2(pMsg); @@ -355,6 +355,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ASSERT(pSyncMsg != NULL); code = syncNodeOnAppendEntriesReplySnapshot2Cb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotSendCb(pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); } else if (pMsg->msgType == TDMT_SYNC_SET_VNODE_STANDBY) { code = vnodeSetStandBy(pVnode); if (code != 0 && terrno != 0) code = terrno; @@ -415,7 +423,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + vTrace("vgId:%d, commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); @@ -430,14 +438,14 @@ static void vnodeSyncCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta c static void vnodeSyncPreCommitMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + vTrace("vgId:%d, pre-commit-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); } static void vnodeSyncRollBackMsg(SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { SVnode *pVnode = pFsm->data; - vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", + vTrace("vgId:%d, rollback-cb is excuted, fsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, msgtype:%d %s", syncGetVgId(pVnode->sync), pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), pMsg->msgType, TMSG_INFO(pMsg->msgType)); } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 9195747bee..bd3402dc39 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -64,7 +64,7 @@ void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { qDebug("db %d vgInfo:", i); for (int32_t j = 0; j < vgNum; ++j) { SVgroupInfo* pInfo = taosArrayGet(pDb, j); - qDebug("vg %d info: vgId:%d", j, pInfo->vgId); + qDebug("vg :%d info: vgId:%d", j, pInfo->vgId); } } } else { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 21e78d4925..ad73fe40d2 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -731,7 +731,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pVgroup = *vgInfo; - ctgDebug("Got tb %s hash vgroup, vgId %d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); CTG_RET(code); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 5df653324d..82387119b6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -51,13 +51,12 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) -#define START_TS_COLUMN_INDEX 0 -#define END_TS_COLUMN_INDEX 1 -#define UID_COLUMN_INDEX 2 -#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX +#define START_TS_COLUMN_INDEX 0 +#define END_TS_COLUMN_INDEX 1 +#define UID_COLUMN_INDEX 2 +#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX #define DELETE_GROUPID_COLUMN_INDEX 2 - enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, @@ -81,8 +80,8 @@ typedef struct SResultInfo { // TODO refactor } SResultInfo; typedef struct STableQueryInfo { - TSKEY lastKey; // last check ts, todo remove it later - SResultRowPosition pos; // current active time window + TSKEY lastKey; // last check ts, todo remove it later + SResultRowPosition pos; // current active time window } STableQueryInfo; typedef struct SLimit { @@ -105,7 +104,7 @@ typedef struct STaskCostInfo { uint64_t loadDataTime; SFileBlockLoadRecorder* pRecoder; - uint64_t elapsedTime; + uint64_t elapsedTime; uint64_t firstStageMergeTime; uint64_t winInfoSize; @@ -118,8 +117,8 @@ typedef struct STaskCostInfo { } STaskCostInfo; typedef struct SOperatorCostInfo { - double openCost; - double totalCost; + double openCost; + double totalCost; } SOperatorCostInfo; struct SOperatorInfo; @@ -139,24 +138,35 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; +typedef struct { + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + void* metaBlk; // for tmq fetching meta + SSDataBlock* pullOverBlk; // for streaming + SWalFilterCond cond; +} SStreamTaskInfo; + typedef struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + + SStreamTaskInfo streamInfo; + struct { - char *tablename; - char *dbname; - int32_t tversion; - SSchemaWrapper*sw; + char* tablename; + char* dbname; + int32_t tversion; + SSchemaWrapper* sw; } schemaVer; - STableListInfo tableqinfoList; // this is a table list - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + STableListInfo tableqinfoList; // this is a table list + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] struct SOperatorInfo* pRoot; } SExecTaskInfo; @@ -168,36 +178,36 @@ enum { }; typedef struct SOperatorFpSet { - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; - __optr_explain_fn_t getExplainFn; + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_explain_fn_t getExplainFn; } SOperatorFpSet; typedef struct SExprSupp { SExprInfo* pExprInfo; - int32_t numOfExprs; // the number of scalar expression in group operator + int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info } SExprSupp; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blocking; // block operator or not - uint8_t status; // denote if current operator is completed - char* name; // name, for debug purpose - void* info; // extension attribution - SExprSupp exprSupp; - SExecTaskInfo* pTaskInfo; - SOperatorCostInfo cost; - SResultInfo resultInfo; - struct SOperatorInfo** pDownstream; // downstram pointer list - int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - SOperatorFpSet fpSet; + uint8_t operatorType; + bool blocking; // block operator or not + uint8_t status; // denote if current operator is completed + char* name; // name, for debug purpose + void* info; // extension attribution + SExprSupp exprSupp; + SExecTaskInfo* pTaskInfo; + SOperatorCostInfo cost; + SResultInfo resultInfo; + struct SOperatorInfo** pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + SOperatorFpSet fpSet; } SOperatorInfo; typedef enum { @@ -210,12 +220,12 @@ typedef enum { #define COL_MATCH_FROM_SLOT_ID 0x2 typedef struct SSourceDataInfo { - int32_t index; - SRetrieveTableRsp* pRsp; - uint64_t totalRows; - int32_t code; - EX_SOURCE_STATUS status; - const char* taskId; + int32_t index; + SRetrieveTableRsp* pRsp; + uint64_t totalRows; + int32_t code; + EX_SOURCE_STATUS status; + const char* taskId; } SSourceDataInfo; typedef struct SLoadRemoteDataInfo { @@ -325,10 +335,10 @@ typedef enum EStreamScanMode { } EStreamScanMode; typedef struct SCatchSupporter { - SHashObj* pWindowHashTable; // quick locate the window object for each window - SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file - int32_t keySize; - int64_t* pKeyBuf; + SHashObj* pWindowHashTable; // quick locate the window object for each window + SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file + int32_t keySize; + int64_t* pKeyBuf; } SCatchSupporter; typedef struct SStreamAggSupporter { @@ -344,48 +354,48 @@ typedef struct SStreamAggSupporter { typedef struct SessionWindowSupporter { SStreamAggSupporter* pStreamAggSup; - int64_t gap; - uint8_t parentType; + int64_t gap; + uint8_t parentType; } SessionWindowSupporter; typedef struct SStreamScanInfo { - uint64_t tableUid; // queried super table uid - SExprInfo* pPseudoExpr; - int32_t numOfPseudoExpr; - int32_t primaryTsIndex; // primary time stamp slot id - SReadHandle readHandle; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. - SArray* pColMatchInfo; // - SNode* pCondition; + uint64_t tableUid; // queried super table uid + SExprInfo* pPseudoExpr; + int32_t numOfPseudoExpr; + int32_t primaryTsIndex; // primary time stamp slot id + SReadHandle readHandle; + SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. + SArray* pColMatchInfo; // + SNode* pCondition; - SArray* pBlockLists; // multiple SSDatablock. - SSDataBlock* pRes; // result SSDataBlock - SSDataBlock* pUpdateRes; // update SSDataBlock - int32_t updateResIndex; - int32_t blockType; // current block type - int32_t validBlockIndex; // Is current data has returned? - uint64_t numOfExec; // execution times - STqReader* tqReader; + SArray* pBlockLists; // multiple SSDatablock. + SSDataBlock* pRes; // result SSDataBlock + SSDataBlock* pUpdateRes; // update SSDataBlock + int32_t updateResIndex; + int32_t blockType; // current block type + int32_t validBlockIndex; // Is current data has returned? + uint64_t numOfExec; // execution times + STqReader* tqReader; - int32_t tsArrayIndex; - SArray* tsArray; - uint64_t groupId; - SUpdateInfo* pUpdateInfo; + int32_t tsArrayIndex; + SArray* tsArray; + uint64_t groupId; + SUpdateInfo* pUpdateInfo; - EStreamScanMode scanMode; - SOperatorInfo* pStreamScanOp; - SOperatorInfo* pTableScanOp; - SArray* childIds; + EStreamScanMode scanMode; + SOperatorInfo* pStreamScanOp; + SOperatorInfo* pTableScanOp; + SArray* childIds; SessionWindowSupporter sessionSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; // for state operator - int32_t pullDataResIndex; - SSDataBlock* pPullDataRes; // pull data SSDataBlock - SSDataBlock* pDeleteDataRes; // delete data SSDataBlock - int32_t deleteDataIndex; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock + SSDataBlock* pDeleteDataRes; // delete data SSDataBlock + int32_t deleteDataIndex; // status for tmq - //SSchemaWrapper schema; + // SSchemaWrapper schema; STqOffset offset; } SStreamScanInfo; @@ -445,7 +455,6 @@ typedef struct SIntervalAggOperatorInfo { int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. STimeWindow win; // query time range bool timeWindowInterpo; // interpolation needed or not - char** pRow; // previous row/tuple of already processed datablock SArray* pInterpCols; // interpolation columns int32_t order; // current SSDataBlock scan order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] @@ -595,7 +604,7 @@ typedef struct SSessionAggOperatorInfo { int64_t gap; // session window gap int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; - SNode *pCondition; + const SNode* pCondition; } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { @@ -657,7 +666,7 @@ typedef struct SStateWindowOperatorInfo { int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; // bool reptScan; - const SNode *pCondition; + const SNode* pCondition; } SStateWindowOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -893,7 +902,7 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t precision, STimeWindow* win); + int32_t precision, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4e5458e620..1f11f4be98 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -323,12 +323,12 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, if (code == TSDB_CODE_INDEX_REBUILDING) { code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); } else if (code != TSDB_CODE_SUCCESS) { - qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); + qError("failed to get tableIds, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); taosArrayDestroy(res); terrno = code; return code; } else { - qDebug("sucess to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); + qDebug("success to get tableIds, size:%d, suid:%" PRIu64, (int)taosArrayGetSize(res), tableUid); } for (int i = 0; i < taosArrayGetSize(res); i++) { @@ -341,7 +341,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } if (code != TSDB_CODE_SUCCESS) { - qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); + qError("failed to get tableIds, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); terrno = code; return code; } @@ -748,11 +748,12 @@ SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; return c; } @@ -774,6 +775,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->suid = pTableScanNode->scan.suid; pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->startVersion = -1; + pCond->endVersion = -1; // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a83565cbe0..fc35fba935 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } - } else if (type == STREAM_INPUT__DATA_SCAN) { + } else if (type == STREAM_INPUT__TABLE_SCAN) { // do nothing - ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN); + ASSERT(pInfo->blockType == STREAM_INPUT__TABLE_SCAN); } else { ASSERT(0); } @@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__TABLE_SCAN, 0, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index baf667d618..ad33c0ae55 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -267,7 +267,46 @@ const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { return &pInfo->offset; } -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + return pTaskInfo->streamInfo.metaBlk; +} + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); + return 0; +} + +int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.prepareStatus = *pOffset; + // TODO: optimize + /*if (pTaskInfo->streamInfo.lastStatus.type != pOffset->type ||*/ + /*pTaskInfo->streamInfo.prepareStatus.version != pTaskInfo->streamInfo.lastStatus.version) {*/ + while (1) { + uint8_t type = pOperator->operatorType; + pOperator->status = OP_OPENED; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOperator->info; + if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) { + return -1; + } + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } + /*}*/ + return 0; +} + +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (uid == 0) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b638fb82e1..4b0f29e106 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2852,7 +2852,7 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pScanInfo = pOperator->info; - pScanInfo->blockType = STREAM_INPUT__DATA_SCAN; + pScanInfo->blockType = STREAM_INPUT__TABLE_SCAN; pScanInfo->pTableScanOp->status = OP_OPENED; @@ -2894,7 +2894,7 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { pInfo->scanTimes = 0; pInfo->curTWinIdx = 0; - qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, + qDebug("tsdb reader offset seek to uid %" PRId64 " ts %" PRId64 ", table cur set to %d , all table num %d", uid, ts, pInfo->currentTable, tableSz); } @@ -3287,7 +3287,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { + // TODO optimize + /*if (pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM) {*/ doSetOperatorCompleted(pOperator); + /*}*/ break; } if (pBlock->info.type == STREAM_RETRIEVE) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5aafaa0b45..cdac46c948 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -39,8 +39,8 @@ static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capac static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -static void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock); +static int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, + SSDataBlock* pBlock, const char* idStr); static bool processBlockWithProbability(const SSampleExecInfo* pInfo); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -210,6 +210,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); @@ -263,7 +264,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { SExprSupp* pSup = &pTableScanInfo->pseudoSup; - addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock); + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } int64_t st = taosGetTimestampMs(); @@ -297,16 +302,21 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow); } -void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock) { +int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, + SSDataBlock* pBlock, const char* idStr) { // currently only the tbname pseudo column if (numOfPseudoExpr == 0) { - return; + return TSDB_CODE_SUCCESS; } SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); - metaGetTableEntryByUid(&mr, pBlock->info.uid); + int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + metaReaderClear(&mr); + return terrno; + } for (int32_t j = 0; j < numOfPseudoExpr; ++j) { SExprInfo* pExpr = &pPseudoExpr[j]; @@ -348,6 +358,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ } metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; } void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) { @@ -595,7 +606,7 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } - + taosMemoryFreeClear(param); } @@ -679,34 +690,46 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* return pOperator; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) { - int32_t rowLen = 0; +static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { + *rowLen = 0; SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); - metaGetTableEntryByUid(&mr, uid); + int32_t code = metaGetTableEntryByUid(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + if (mr.me.type == TSDB_SUPER_TABLE) { int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; - metaGetTableEntryByUid(&mr, suid); + code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_NORMAL_TABLE) { int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; } } metaReaderClear(&mr); - return rowLen; + return TSDB_CODE_SUCCESS; } static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { @@ -715,9 +738,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { } SBlockDistInfo* pBlockScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - blockDistInfo.rowSize = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid); + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); @@ -745,7 +772,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - + taosMemoryFreeClear(param); } @@ -911,7 +938,7 @@ static bool prepareDataScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t t setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, *pRowIndex, gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, NULL); + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[*pRowIndex], &pInfo->interval, pInfo->interval.precision, TSDB_ORDER_ASC); setGroupId(pInfo, pSDB, GROUPID_COLUMN_INDEX, *pRowIndex); (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, *pRowIndex, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); @@ -1130,15 +1157,124 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32 uidCol[i] = getGroupId(pOperator, uidCol[i]); } } +static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock) { + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + SOperatorInfo* pOperator = pInfo->pStreamScanOp; + SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo; + + pInfo->pRes->info.rows = pBlock->info.rows; + pInfo->pRes->info.uid = pBlock->info.uid; + pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = pBlock->info.rows; + + // for generating rollup SMA result, each time is an independent time serie. + // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this + if (pInfo->assignBlockUid) { + pInfo->pRes->info.groupId = pBlock->info.uid; + } + + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupIdPre) { + pInfo->pRes->info.groupId = *groupIdPre; + } else { + pInfo->pRes->info.groupId = 0; + } + + // todo extract method + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + + bool colExists = false; + for (int32_t j = 0; j < blockDataGetNumOfCols(pBlock); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(pBlock, j); + if (pResCol->info.colId == pColMatchInfo->colId) { + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); + colExists = true; + break; + } + } + + // the required column does not exists in submit block, let's set it to be all null value + if (!colExists) { + SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); + colDataAppendNNULL(pDst, 0, pBlockInfo->rows); + } + } + + taosArrayDestroy(pBlock->pDataBlock); + + ASSERT(pInfo->pRes->pDataBlock != NULL); +#if 0 + if (pInfo->pRes->pDataBlock == NULL) { + // TODO add log + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); + pOperator->status = OP_EXEC_DONE; + pTaskInfo->code = terrno; + return -1; + } +#endif + + // currently only the tbname pseudo column + if (pInfo->numOfPseudoExpr > 0) { + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + } + + doFilter(pInfo->pCondition, pInfo->pRes); + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); + if (pBlockInfo->rows > 0) { + return 0; + } + return 0; +} static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { - return NULL; + /*pTaskInfo->code = pOperator->fpSet._openFn(pOperator);*/ + /*if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) {*/ + /*return NULL;*/ + /*}*/ + + if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { + while (1) { + SFetchRet ret = {0}; + tqNextBlock(pInfo->tqReader, &ret); + if (ret.fetchType == FETCH_TYPE__DATA) { + blockDataCleanup(pInfo->pRes); + if (setBlockIntoRes(pInfo, &ret.data) < 0) { + ASSERT(0); + } + /*pTaskInfo->streamInfo.lastStatus = ret.offset;*/ + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + /*} else {*/ + /*tDeleteSSDataBlock(&ret.data);*/ + } + } else if (ret.fetchType == FETCH_TYPE__META) { + ASSERT(0); + pTaskInfo->streamInfo.lastStatus = ret.offset; + pTaskInfo->streamInfo.metaBlk = ret.meta; + return NULL; + } else if (ret.fetchType == FETCH_TYPE__NONE) { + if (ret.offset.version == -1) { + pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; + pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1; + } else { + pTaskInfo->streamInfo.lastStatus = ret.offset; + } + return NULL; + } else { + ASSERT(0); + } + } } size_t total = taosArrayGetSize(pInfo->pBlockLists); @@ -1146,7 +1282,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { /*doClearBufferedBlocks(pInfo);*/ - pOperator->status = OP_EXEC_DONE; + /*pOperator->status = OP_EXEC_DONE;*/ return NULL; } @@ -1255,8 +1391,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.capacity = block.info.rows; - - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; @@ -1295,6 +1429,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } taosArrayDestroy(block.pDataBlock); + + ASSERT(pInfo->pRes->pDataBlock != NULL); +#if 0 if (pInfo->pRes->pDataBlock == NULL) { // TODO add log updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); @@ -1302,10 +1439,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->code = terrno; return NULL; } +#endif // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); + code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } doFilter(pInfo->pCondition, pInfo->pRes); @@ -1321,7 +1462,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pBlockInfo->rows == 0) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); - pOperator->status = OP_EXEC_DONE; + /*pOperator->status = OP_EXEC_DONE;*/ } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); @@ -1339,7 +1480,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) { + } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) { // check reader last status // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); @@ -1730,7 +1871,16 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SMetaReader mr = {0}; metaReaderInit(&mr, pInfo->readHandle.meta, 0); - metaGetTableEntryByUid(&mr, pInfo->pCur->mr.me.ctbEntry.suid); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); @@ -2125,7 +2275,12 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { STableKeyInfo* item = taosArrayGet(pInfo->pTableList->pTableList, pInfo->curPos); - metaGetTableEntryByUid(&mr, item->uid); + int32_t code = metaGetTableEntryByUid(&mr, item->uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + longjmp(pTaskInfo->env, terrno); + } for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); @@ -2177,7 +2332,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); - + taosMemoryFreeClear(param); } @@ -2410,8 +2565,11 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // currently only the tbname pseudo column if (pTableScanInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr, - pBlock); + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr, + pBlock, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } int64_t st = taosGetTimestampMs(); @@ -2669,7 +2827,7 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); - + taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0847b0267b..f9810913e3 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -79,36 +79,111 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } } +static STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order) { + int32_t factor = (order == TSDB_ORDER_ASC)? -1:1; + + STimeWindow win = *pWindow; + STimeWindow save = win; + while(win.skey <= ts && win.ekey >= ts) { + save = win; + win.skey = taosTimeAdd(win.skey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + win.ekey = taosTimeAdd(win.ekey, factor * pInterval->sliding, pInterval->slidingUnit, pInterval->precision); + } + + return save; +} + +// todo do refactor // get the correct time window according to the handled timestamp STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t precision, STimeWindow* win) { + int32_t precision, int32_t order) { STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value getInitialStartTimeWindow(pInterval, precision, ts, &w, true); w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; - } else { - w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + return w; } - if (w.skey > ts || w.ekey < ts) { - if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, pInterval, precision); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + w = getResultRowByPos(pBuf, &pResultRowInfo->cur)->win; + + if (pInterval->interval == pInterval->sliding) { + if (w.skey > ts || w.ekey < ts) { + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } + } + } else { // it is an sliding window query, in which sliding value is not equalled to + // interval value, and we need to find the first qualified time window for asc/desc traverse respectively. + if (order == TSDB_ORDER_ASC) { + if (w.skey <= ts && w.ekey >= ts) { + // ts is resident in current time window, but we need to find the first + //qualified time window that cover this timestamp + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } else { + // todo refactor: + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; + + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } + } } else { - int64_t st = w.skey; + if (w.skey <= ts && w.ekey >= ts) { + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } else { + // todo refactor: + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + } else { + int64_t st = w.skey; - if (st > ts) { - st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + if (st > ts) { + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + int64_t et = st + pInterval->interval - 1; + if (et < ts) { + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + } + + w.skey = st; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; + + w = getFirstQualifiedTimeWindow(ts, &w, pInterval, order); + } } - - int64_t et = st + pInterval->interval - 1; - if (et < ts) { - st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; - } - - w.skey = st; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } } return w; @@ -856,7 +931,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, &pInfo->win); + pInfo->interval.precision, pInfo->order); int32_t ret = TSDB_CODE_SUCCESS; if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) { ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, @@ -1010,21 +1085,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, scanFlag, true); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); - -#if 0 // test for encode/decode result info - if(pOperator->fpSet.encodeResultRow){ - char *result = NULL; - int32_t length = 0; - SAggSupporter *pSup = &pInfo->aggSup; - pOperator->fpSet.encodeResultRow(pOperator, &result, &length); - taosHashClear(pSup->pResultRowHashTable); - pInfo->binfo.resultRowInfo.size = 0; - pOperator->fpSet.decodeResultRow(pOperator, result); - if(result){ - taosMemoryFree(result); - } - } -#endif } closeAllResultRows(&pInfo->binfo.resultRowInfo); @@ -1330,7 +1390,7 @@ void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); if (pUpWins) { SWinRes winRes = {.ts = win.skey, .groupId = groupIds[i]}; @@ -1352,7 +1412,7 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, TSDB_ORDER_ASC); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); uint64_t winGpId = pGpDatas ? pGpDatas[i] : pBlock->info.groupId; bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), winGpId, numOfOutput); @@ -1392,7 +1452,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, TSDB_ORDER_ASC); SWinRes winRe = { .ts = win.skey, .groupId = groupId, @@ -1402,13 +1462,13 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, if (chIds && pPullDataMap) { SArray* chAy = *(SArray**)chIds; int32_t size = taosArrayGetSize(chAy); - qInfo("======window %ld wait child size:%d", win.skey, size); + qInfo("window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qInfo("======window %ld wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qInfo("window %" PRId64 " wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { - qInfo("======close window %ld", win.skey); + qInfo("close window %" PRId64, win.skey); } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -2488,7 +2548,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, - pInfo->interval.precision, NULL); + pInfo->interval.precision, pInfo->order); while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if (pInfo->ignoreExpiredData && isClosed) { @@ -4454,8 +4514,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR currTs = tsCols[currPos]; currWin.skey = currTs; currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, - iaInfo->interval.precision) - - 1; + iaInfo->interval.precision) - 1; startPos = currPos; ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); @@ -4692,7 +4751,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, - iaInfo->interval.precision, &iaInfo->win); + iaInfo->interval.precision, iaInfo->order); int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 3c12889aca..bba4b254c5 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -1029,7 +1029,7 @@ TEST(testCase, external_sort_Test) { int64_t e = taosGetTimestampUs(); if (t++ == 1) { - printf("---------------elapsed:%ld\n", e - s); + printf("---------------elapsed:%" PRId64 "\n", e - s); } if (pRes == NULL) { @@ -1046,7 +1046,7 @@ TEST(testCase, external_sort_Test) { } int64_t s2 = taosGetTimestampUs(); - printf("total:%ld\n", s2 - s1); + printf("total:%" PRId64 "\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); taosMemoryFreeClear(exp); @@ -1101,7 +1101,7 @@ TEST(testCase, sorted_merge_Test) { int64_t e = taosGetTimestampUs(); if (t++ == 1) { - printf("---------------elapsed:%ld\n", e - s); + printf("---------------elapsed:%" PRId64 "\n", e - s); } if (pRes == NULL) { @@ -1112,13 +1112,13 @@ TEST(testCase, sorted_merge_Test) { // SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); for (int32_t i = 0; i < pRes->info.rows; ++i) { // char* p = colDataGetData(pCol2, i); - printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); + printf("%d: %" PRId64 "\n", total++, ((int64_t*)pCol1->pData)[i]); // printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); } } int64_t s2 = taosGetTimestampUs(); - printf("total:%ld\n", s2 - s1); + printf("total:%" PRId64 "\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); taosMemoryFreeClear(exp); @@ -1179,7 +1179,7 @@ TEST(testCase, time_interval_Operator_Test) { int64_t e = taosGetTimestampUs(); if (t++ == 1) { - printf("---------------elapsed:%ld\n", e - s); + printf("---------------elapsed:%" PRId64 "\n", e - s); } if (pRes == NULL) { @@ -1190,13 +1190,13 @@ TEST(testCase, time_interval_Operator_Test) { // SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); for (int32_t i = 0; i < pRes->info.rows; ++i) { // char* p = colDataGetData(pCol2, i); - printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); + printf("%d: %" PRId64 "\n", total++, ((int64_t*)pCol1->pData)[i]); // printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); } } int64_t s2 = taosGetTimestampUs(); - printf("total:%ld\n", s2 - s1); + printf("total:%" PRId64 "\n", s2 - s1); pOperator->closeFn(pOperator->info, 2); taosMemoryFreeClear(exp); diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 70def78263..6e244152f2 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -118,7 +118,7 @@ SSDataBlock* getSingleColDummyBlock(void* param) { } colDataAppend(pColInfo, i, result, false); - printf("int: %ld\n", v); + printf("int: %" PRId64 "\n", v); taosMemoryFree(result); } } @@ -333,7 +333,7 @@ TEST(testCase, external_mem_sort_Test) { }else{ memcpy((char*)(&result) + sizeof(int64_t) - tDataTypes[pInfo[i].type].bytes, v, tDataTypes[pInfo[i].type].bytes); } - printf("%d: %ld\n", row++, result); + printf("%d: %" PRId64 "\n", row++, result); } } taosArrayDestroy(orderInfo); diff --git a/source/libs/function/inc/fnLog.h b/source/libs/function/inc/fnLog.h index d8dd1d1e67..9658b6b782 100644 --- a/source/libs/function/inc/fnLog.h +++ b/source/libs/function/inc/fnLog.h @@ -14,7 +14,7 @@ extern "C" { #define fnFatal(...) { if (udfDebugFlag & DEBUG_FATAL) { taosPrintLog("UDF FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} #define fnError(...) { if (udfDebugFlag & DEBUG_ERROR) { taosPrintLog("UDF ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} #define fnWarn(...) { if (udfDebugFlag & DEBUG_WARN) { taosPrintLog("UDF WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define fnInfo(...) { if (udfDebugFlag & DEBUG_INFO) { taosPrintLog("UDF ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define fnInfo(...) { if (udfDebugFlag & DEBUG_INFO) { taosPrintLog("UDF ", DEBUG_INFO, 255, __VA_ARGS__); }} #define fnDebug(...) { if (udfDebugFlag & DEBUG_DEBUG) { taosPrintLog("UDF ", DEBUG_DEBUG, udfDebugFlag, __VA_ARGS__); }} #define fnTrace(...) { if (udfDebugFlag & DEBUG_TRACE) { taosPrintLog("UDF ", DEBUG_TRACE, udfDebugFlag, __VA_ARGS__); }} // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index d8fcca30c0..1706c129ce 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3467,6 +3467,10 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS setBufPageDirty(pPage, true); releaseBufPage(pCtx->pBuf, pPage); +#ifdef BUF_PAGE_DEBUG + qDebug("page_saveTuple pos:%p,pageId:%d, offset:%d\n", pPos, pPos->pageId, + pPos->offset); +#endif } void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { @@ -3501,6 +3505,9 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS setBufPageDirty(pPage, true); releaseBufPage(pCtx->pBuf, pPage); +#ifdef BUF_PAGE_DEBUG + qDebug("page_copyTuple pos:%p, pageId:%d, offset:%d", pPos, pPos->pageId, pPos->offset); +#endif } int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index dcf6676f9f..c16c3e3937 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -203,7 +203,7 @@ int32_t udfStartUdfd(int32_t startDnodeId) { uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); pData->needCleanUp = false; - fnInfo("dnode udfd cleaned up after spawn err"); + fnInfo("udfd is cleaned up after spawn err"); } else { pData->needCleanUp = true; } @@ -212,7 +212,7 @@ int32_t udfStartUdfd(int32_t startDnodeId) { int32_t udfStopUdfd() { SUdfdData *pData = &udfdGlobal; - fnInfo("dnode to stop udfd. need cleanup: %d, spawn err: %d", + fnInfo("udfd start to stop, need cleanup:%d, spawn err:%d", pData->needCleanUp, pData->spawnErr); if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) { return 0; @@ -225,7 +225,7 @@ int32_t udfStopUdfd() { #ifdef WINDOWS if (pData->jobHandle != NULL) CloseHandle(pData->jobHandle); #endif - fnInfo("dnode udfd cleaned up"); + fnInfo("udfd is cleaned up"); return 0; } @@ -467,7 +467,7 @@ int32_t getUdfdPipeName(char* pipeName, int32_t size) { size_t dnodeIdSize = sizeof(dnodeId); int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); if (err != 0) { - fnError("get dnode id from env. error: %s.", uv_err_name(err)); + fnError("failed to get dnodeId from env since %s", uv_err_name(err)); dnodeId[0] = '1'; } #ifdef _WIN32 @@ -475,7 +475,7 @@ int32_t getUdfdPipeName(char* pipeName, int32_t size) { #else snprintf(pipeName, size, "%s/%s%s", tsDataDir, UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); #endif - fnInfo("get dnode id from env. dnode id: %s. pipe path: %s", dnodeId, pipeName); + fnInfo("get dnodeId:%s from env, pipe path:%s", dnodeId, pipeName); return 0; } @@ -1609,7 +1609,7 @@ int32_t udfcClose() { taosArrayDestroy(udfc->udfStubs); uv_mutex_destroy(&udfc->udfStubsMutex); udfc->udfcState = UDFC_STATE_INITAL; - fnInfo("udfc cleaned up"); + fnInfo("udfc is cleaned up"); return 0; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 0fbeac47e6..67133d0bf1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -77,14 +77,6 @@ static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { return TSDB_CODE_SUCCESS; } -static SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { - pName->type = TSDB_TABLE_NAME_T; - pName->acctId = acctId; - strcpy(pName->dbname, pDbName); - strcpy(pName->tname, pTableName); - return pName; -} - static int32_t collectUseDatabaseImpl(const char* pFullDbName, SHashObj* pDbs) { SFullDatabaseName name = {0}; strcpy(name.fullDbName, pFullDbName); @@ -5370,7 +5362,8 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch return TSDB_CODE_SUCCESS; } -static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { +static void destroyCreateTbReqBatch(void* data) { + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); @@ -5387,7 +5380,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { +int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { SVnodeModifOpStmt* pNewStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (pNewStmt == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -5453,10 +5446,10 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, const STag* pTag, uint64_t suid, SVgroupInfo* pVgInfo) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; - strcpy(name.dbname, pStmt->dbName); - tNameGetFullDbName(&name, dbFName); +// char dbFName[TSDB_DB_FNAME_LEN] = {0}; +// SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; +// strcpy(name.dbname, pStmt->dbName); +// tNameGetFullDbName(&name, dbFName); struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; @@ -5717,7 +5710,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla return code; } -static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { +SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { return NULL; @@ -5732,7 +5725,6 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } serializeVgroupCreateTableBatch(pTbBatch, pBufArray); - destroyCreateTbReqBatch(pTbBatch); } while (true); return pBufArray; @@ -5746,6 +5738,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) return TSDB_CODE_OUT_OF_MEMORY; } + taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pStmt->pSubTables) { @@ -5757,7 +5750,7 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) } } - SArray* pBufArray = serializeVgroupsCreateTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); taosHashCleanup(pVgroupHashmap); if (NULL == pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; @@ -5817,7 +5810,10 @@ over: return code; } -static void destroyDropTbReqBatch(SVgroupDropTableBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } +static void destroyDropTbReqBatch(void* data) { + SVgroupDropTableBatch* pTbBatch = (SVgroupDropTableBatch*)data; + taosArrayDestroy(pTbBatch->req.pArray); +} static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SArray* pBufArray) { int tlen; @@ -5851,7 +5847,7 @@ static int32_t serializeVgroupDropTableBatch(SVgroupDropTableBatch* pTbBatch, SA return TSDB_CODE_SUCCESS; } -static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupHashmap) { +SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap) { SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*)); if (NULL == pBufArray) { return NULL; @@ -5866,7 +5862,6 @@ static SArray* serializeVgroupsDropTableBatch(int32_t acctId, SHashObj* pVgroupH } serializeVgroupDropTableBatch(pTbBatch, pBufArray); - destroyDropTbReqBatch(pTbBatch); } while (true); return pBufArray; @@ -5880,6 +5875,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } + taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); bool isSuperTable = false; SNode* pNode; FOREACH(pNode, pStmt->pTables) { @@ -5898,7 +5894,7 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } - SArray* pBufArray = serializeVgroupsDropTableBatch(pCxt->pParseCxt->acctId, pVgroupHashmap); + SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); taosHashCleanup(pVgroupHashmap); if (NULL == pBufArray) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 908149c3ea..ed8786170d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -381,11 +381,11 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId %d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 - " tagNum %d colNum %d precision %d rowSize %d", - msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, pTableMeta->suid, - pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags, pTableMeta->tableInfo.numOfColumns, - pTableMeta->tableInfo.precision, pTableMeta->tableInfo.rowSize); + qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 + " tagNum %d colNum %d precision %d rowSize %d", + msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, + pTableMeta->suid, pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags, + pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.precision, pTableMeta->tableInfo.rowSize); *pMeta = pTableMeta; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index c2d984c7dd..1794d8f5ed 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); +int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a7f66ebb7d..57819db0b8 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3827,13 +3827,20 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); + int32_t code = sclCreateColumnInfoData(&type, pSrc->info.rows, &output); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = (int8_t *)output.columnData->pData; + *p = taosMemoryMalloc(output.numOfRows * sizeof(bool)); + + memcpy(*p, output.columnData->pData, output.numOfRows); + colDataDestroy(output.columnData); + taosMemoryFree(output.columnData); taosArrayDestroy(pList); return false; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cbb1089d61..2cbe1e5c96 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -35,12 +35,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { return TSDB_CODE_SUCCESS; } - -SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { +int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return terrno; } pColumnData->info.type = pType->type; @@ -52,19 +51,25 @@ SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); - return NULL; - } else { - return pColumnData; + return terrno; } + + pParam->columnData = pColumnData; + pParam->type = SHOULD_FREE_COLDATA; + return TSDB_CODE_SUCCESS; } int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); + int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - int32_t code = vectorConvertImpl(&in, out); + code = vectorConvertImpl(&in, out); sclFreeParam(&in); return code; @@ -157,7 +162,7 @@ void sclFreeRes(SHashObj *res) { void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); - taosMemoryFree(param->columnData); + taosMemoryFreeClear(param->columnData); } if (param->pHashFilter != NULL) { @@ -190,8 +195,9 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; + ASSERT(param->columnData == NULL); param->numOfRows = 1; - param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); + /*int32_t code = */sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataAppendNULL(param->columnData, 0); } else { @@ -429,10 +435,9 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&node->node.resType, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); } code = (*ffpSet.process)(params, paramNum, output); @@ -482,10 +487,9 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = sclCreateColumnInfoData(&t, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&t, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); } bool value = false; @@ -537,18 +541,19 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; // json not support in in operator - if(nodeType(node->pLeft) == QUERY_NODE_VALUE){ + if (nodeType(node->pLeft) == QUERY_NODE_VALUE) { SValueNode *valueNode = (SValueNode *)node->pLeft; - if(valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)){ + if (valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)) { SCL_RET(TSDB_CODE_QRY_JSON_IN_ERROR); } } SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { - sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&node->node.resType, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); + } } _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); @@ -563,7 +568,10 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _return: for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParam(¶ms[i]); + if (params[i].type == SHOULD_FREE_COLDATA) { + colDataDestroy(params[i].columnData); + taosMemoryFreeClear(params[i].columnData); + } } taosMemoryFreeClear(params); @@ -766,7 +774,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { return sclRewriteNonConstOperator(pNode, ctx); } - SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; + SScalarParam output = {0}; ctx->code = sclExecOperator(node, ctx, &output); if (ctx->code) { return DEAL_RES_ERROR; @@ -834,6 +842,7 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -868,6 +877,7 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -1026,7 +1036,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; } - + + sclFreeParam(res); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 18bbacd5b7..98cb923b49 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -865,12 +865,11 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { } int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) { - int32_t code = 0; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); - if (output->columnData == NULL) { + int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); + if (code != TSDB_CODE_SUCCESS) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -940,13 +939,12 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); - if (pDest->columnData == NULL) { - sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = sclCreateColumnInfoData(&t, pParam->numOfRows, pDest); + if (code != TSDB_CODE_SUCCESS) { + return code; } - int32_t code = vectorConvertImpl(pParam, pDest); + code = vectorConvertImpl(pParam, pDest); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 8f9332cef8..d8c64eef55 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -205,13 +205,19 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { *pNode = (SNode *)lnode; } +void initScalarParam(SScalarParam* pParam) { + memset(pParam, 0, sizeof(SScalarParam)); + pParam->type = SHOULD_FREE_COLDATA; +} } TEST(timerangeTest, greater) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); + int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -234,7 +240,8 @@ TEST(timerangeTest, greater) { TEST(timerangeTest, greater_and_lower) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -265,7 +272,8 @@ TEST(timerangeTest, greater_and_lower) { TEST(timerangeTest, greater_equal_and_lower_equal) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -297,7 +305,8 @@ TEST(timerangeTest, greater_equal_and_lower_equal) { TEST(timerangeTest, greater_and_lower_not_strict) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode1 = NULL, *logicNode2 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall1 = 222, tbig1 = 333; int64_t tsmall2 = 444, tbig2 = 555; SNode *list[2] = {0}; @@ -350,7 +359,8 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -405,7 +415,8 @@ TEST(columnTest, int_column_greater_smallint_value) { int16_t rightv= 4; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); @@ -460,7 +471,8 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); @@ -503,7 +515,8 @@ TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -567,7 +580,8 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -614,7 +628,8 @@ TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {false, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -661,7 +676,8 @@ TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {true, true, true, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -710,7 +726,8 @@ TEST(opTest, smallint_column_greater_int_column) { int32_t rightv[5]= {0, -5, -4, 23, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); @@ -747,7 +764,8 @@ TEST(opTest, smallint_value_add_int_column) { int16_t rightv[5]= {0, -1, -4, -1, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); @@ -790,7 +808,8 @@ TEST(opTest, bigint_column_multi_binary_column) { } bool eRes[5] = {false, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -833,7 +852,8 @@ TEST(opTest, smallint_column_and_binary_column) { } bool eRes[5] = {false, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -871,7 +891,8 @@ TEST(opTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; bool eRes[5] = {true, true, false, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); @@ -909,7 +930,8 @@ TEST(opTest, smallint_column_or_double_value) { double rightv= 10.2; bool eRes[5] = {true, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -945,7 +967,8 @@ TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 07440e7435..9b40f0a465 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1092,7 +1092,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do printf("op:%s,1result:%f,except:%f\n", gOptrStr[opType].str, *((double *)colDataGetData(column, 0)), exceptValue); ASSERT_TRUE(fabs(*((double *)colDataGetData(column, 0)) - exceptValue) < 0.0001); }else if(opType == OP_TYPE_BIT_AND || opType == OP_TYPE_BIT_OR){ - printf("op:%s,2result:%ld,except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue); + printf("op:%s,2result:%" PRId64 ",except:%f\n", gOptrStr[opType].str, *((int64_t *)colDataGetData(column, 0)), exceptValue); ASSERT_EQ(*((int64_t *)colDataGetData(column, 0)), exceptValue); }else if(opType == OP_TYPE_GREATER_THAN || opType == OP_TYPE_GREATER_EQUAL || opType == OP_TYPE_LOWER_THAN || opType == OP_TYPE_LOWER_EQUAL || opType == OP_TYPE_EQUAL || opType == OP_TYPE_NOT_EQUAL || diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index e43c1e4b33..8409e1c711 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -131,7 +131,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); if (pEntry == NULL) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); + snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%" PRId64 ", since %s", pMsg->prevLogIndex, terrstr()); syncNodeErrorLog(ths, logBuf); return -1; } @@ -150,7 +150,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, reject, pre-index:%ld, pre-term:%lu, datalen:%d", + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, reject, pre-index:%" PRId64 ", pre-term:%" PRIu64 ", datalen:%d", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); } while (0); @@ -167,10 +168,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); SRpcMsg rpcMsg; @@ -194,7 +194,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, accept, pre-index:%ld, pre-term:%lu, datalen:%d", + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, accept, pre-index:%" PRId64 ", pre-term:%" PRIu64 ", datalen:%d", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); } while (0); @@ -207,7 +208,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); if (pExtraEntry == NULL) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); + snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%" PRId64 ", since %s", extraIndex, terrstr()); syncNodeErrorLog(ths, logBuf); return -1; } @@ -229,7 +230,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); SyncIndex delEnd = extraIndex; - sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%" PRId64 ", delEnd:%" PRId64, conflict, delBegin, + delEnd); // notice! reverse roll back! for (SyncIndex index = delEnd; index >= delBegin; --index) { @@ -237,7 +239,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); if (pRollBackEntry == NULL) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); + snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%" PRId64 ", since %s", index, terrstr()); syncNodeErrorLog(ths, logBuf); return -1; } @@ -350,10 +352,9 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); SRpcMsg rpcMsg; @@ -421,7 +422,7 @@ static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { ASSERT(code == 0); char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "log truncate, from %ld to %ld", delBegin, delEnd); + snprintf(eventLog, sizeof(eventLog), "log truncate, from %" PRId64 " to %" PRId64, delBegin, delEnd); syncNodeEventLog(ths, eventLog); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); @@ -466,7 +467,7 @@ static int32_t syncNodeDoMakeLogSame(SSyncNode* ths, SyncIndex FromIndex) { ASSERT(code == 0); char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "log truncate, from %ld to %ld", delBegin, delEnd); + snprintf(eventLog, sizeof(eventLog), "log truncate, from %" PRId64 " to %" PRId64, delBegin, delEnd); syncNodeEventLog(ths, eventLog); logStoreSimpleLog2("after syncNodeMakeLogSame", ths->pLogStore); @@ -499,13 +500,13 @@ static bool syncNodeOnAppendEntriesBatchLogOK(SSyncNode* pSyncNode, SyncAppendEn SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -513,7 +514,7 @@ static bool syncNodeOnAppendEntriesBatchLogOK(SSyncNode* pSyncNode, SyncAppendEn return true; } - sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -526,13 +527,13 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); if (pMsg->prevLogIndex > myLastIndex) { - sDebug("vgId:%d sync log not ok, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); if (myPreLogTerm == SYNC_TERM_INVALID) { - sDebug("vgId:%d sync log not ok2, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok2, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -540,7 +541,7 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries return true; } - sDebug("vgId:%d sync log not ok3, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + sDebug("vgId:%d sync log not ok3, preindex:%" PRId64, pSyncNode->vgId, pMsg->prevLogIndex); return false; } @@ -597,7 +598,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, fake match2, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + "recv sync-append-entries-batch, fake match2, {pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", datalen:%d, datacount:%d}", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -657,10 +659,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -697,7 +698,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, not match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + "recv sync-append-entries-batch, not match, {pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", datalen:%d, datacount:%d}", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -716,10 +718,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -754,7 +755,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch, match, {pre-index:%ld, pre-term:%lu, datalen:%d, datacount:%d}", + "recv sync-append-entries-batch, match, {pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", datalen:%d, datacount:%d}", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen, pMsg->dataCount); syncNodeEventLog(ths, logBuf); } while (0); @@ -800,10 +802,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -825,8 +826,8 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc ths->commitIndex = snapshot.lastApplyIndex; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%ld to index:%ld", commitBegin, - commitEnd); + snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, + commitBegin, commitEnd); syncNodeEventLog(ths, eventLog); } @@ -927,7 +928,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (condition) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%ld, pre-term:%lu", + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, fake match, pre-index:%" PRId64 ", pre-term:%" PRIu64, pMsg->prevLogIndex, pMsg->prevLogTerm); syncNodeEventLog(ths, logBuf); @@ -967,8 +968,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries, fake match2, pre-index:%ld, pre-term:%lu, datalen:%d", pMsg->prevLogIndex, - pMsg->prevLogTerm, pMsg->dataLen); + "recv sync-append-entries, fake match2, pre-index:%" PRId64 ", pre-term:%" PRIu64 ", datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); } while (0); @@ -1020,10 +1021,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -1058,7 +1058,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs if (condition) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, not match, pre-index:%ld, pre-term:%lu, datalen:%d", + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, not match, pre-index:%" PRId64 ", pre-term:%" PRIu64 ", datalen:%d", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); @@ -1076,10 +1077,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -1111,7 +1111,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs bool hasAppendEntries = pMsg->dataLen > 0; char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, match, pre-index:%ld, pre-term:%lu, datalen:%d", + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries, match, pre-index:%" PRId64 ", pre-term:%" PRIu64 ", datalen:%d", pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); syncNodeEventLog(ths, logBuf); @@ -1152,10 +1153,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs char host[128]; uint16_t port; syncUtilU642Addr(pReply->destId.addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-reply to %s:%d, {term:%lu, pterm:%lu, success:%d, " - "match-index:%ld}", - ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); + sDebug("vgId:%d, send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 + ", success:%d, match-index:%" PRId64 "}", + ths->vgId, host, port, pReply->term, pReply->privateTerm, pReply->success, pReply->matchIndex); } while (0); // send response @@ -1177,8 +1177,8 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->commitIndex = snapshot.lastApplyIndex; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%ld to index:%ld", commitBegin, - commitEnd); + snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, + commitBegin, commitEnd); syncNodeEventLog(ths, eventLog); } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 3f19b6759e..153f18da86 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -52,7 +52,8 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%" PRIu64 ", drop stale response", + pMsg->term); syncNodeEventLog(ths, logBuf); return 0; } @@ -70,7 +71,7 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%" PRIu64, pMsg->term); syncNodeErrorLog(ths, logBuf); return -1; } @@ -155,7 +156,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%" PRIu64 ", drop stale response", + pMsg->term); syncNodeEventLog(ths, logBuf); return -1; } @@ -163,7 +165,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie // error term if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%" PRIu64, pMsg->term); syncNodeErrorLog(ths, logBuf); return -1; } @@ -193,9 +195,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie // start snapshot SSnapshot oldSnapshot; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); - ASSERT(oldSnapshot.lastApplyIndex >= newMatchIndex + 1); - syncNodeStartSnapshotOnce(ths, newMatchIndex + 1, oldSnapshot.lastApplyIndex, oldSnapshot.lastApplyTerm, - pMsg); // term maybe not ok? + if (oldSnapshot.lastApplyIndex > newMatchIndex) { + syncNodeStartSnapshotOnce(ths, newMatchIndex + 1, oldSnapshot.lastApplyIndex, oldSnapshot.lastApplyTerm, + pMsg); // term maybe not ok? + } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), oldSnapshot.lastApplyIndex + 1); syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), newMatchIndex); @@ -208,8 +211,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "reset next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, - host, port); + snprintf(logBuf, sizeof(logBuf), "reset next-index:%" PRId64 ", match-index:%" PRId64 " for %s:%d", newNextIndex, + newMatchIndex, host, port); syncNodeEventLog(ths, logBuf); } while (0); @@ -262,8 +265,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SyncIndex newNextIndex = nextIndex; SyncIndex newMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%ld, match-index:%ld for %s:%d", newNextIndex, newMatchIndex, - host, port); + snprintf(logBuf, sizeof(logBuf), "reset2 next-index:%" PRId64 ", match-index:%" PRId64 " for %s:%d", newNextIndex, + newMatchIndex, host, port); syncNodeEventLog(ths, logBuf); } while (0); @@ -287,7 +290,8 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%" PRIu64 ", drop stale response", + pMsg->term); syncNodeEventLog(ths, logBuf); return 0; } @@ -305,7 +309,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%" PRIu64, pMsg->term); syncNodeErrorLog(ths, logBuf); return -1; } @@ -317,7 +321,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); if (gRaftDetailLog) { - sTrace("update next match, index:%ld, success:%d", pMsg->matchIndex + 1, pMsg->success); + sTrace("update next match, index:%" PRId64 ", success:%d", pMsg->matchIndex + 1, pMsg->success); } // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] @@ -331,7 +335,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries } else { SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); if (gRaftDetailLog) { - sTrace("update next index not match, begin, index:%ld, success:%d", nextIndex, pMsg->success); + sTrace("update next index not match, begin, index:%" PRId64 ", success:%d", nextIndex, pMsg->success); } // notice! int64, uint64 @@ -375,7 +379,7 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); if (gRaftDetailLog) { - sTrace("update next index not match, end, index:%ld, success:%d", nextIndex, pMsg->success); + sTrace("update next index not match, end, index:%" PRId64 ", success:%d", nextIndex, pMsg->success); } } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 982d37826a..f7bee01030 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -57,8 +57,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->commitIndex = snapshot.lastApplyIndex; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%ld to index:%ld", pSyncNode->commitIndex, - snapshot.lastApplyIndex); + snprintf(eventLog, sizeof(eventLog), "commit by snapshot from index:%" PRId64 " to index:%" PRId64, commitBegin, + commitEnd); syncNodeEventLog(pSyncNode, eventLog); } @@ -68,8 +68,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { bool agree = syncAgree(pSyncNode, index); if (gRaftDetailLog) { - sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%ld, pSyncNode->commitIndex:%ld", agree, index, - pSyncNode->commitIndex); + sTrace("syncMaybeAdvanceCommitIndex syncAgree:%d, index:%" PRId64 ", pSyncNode->commitIndex:%" PRId64, agree, + index, pSyncNode->commitIndex); } if (agree) { @@ -83,7 +83,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { newCommitIndex = index; if (gRaftDetailLog) { - sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%ld commit, pSyncNode->commitIndex:%ld", + sTrace("syncMaybeAdvanceCommitIndex maybe to update, newCommitIndex:%" PRId64 " commit, pSyncNode->commitIndex:%" PRId64, newCommitIndex, pSyncNode->commitIndex); } @@ -91,10 +91,9 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { break; } else { if (gRaftDetailLog) { - sTrace( - "syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%lu, " - "pSyncNode->pRaftStore->currentTerm:%lu", - pEntry->term, pSyncNode->pRaftStore->currentTerm); + sTrace("syncMaybeAdvanceCommitIndex can not commit due to term not equal, pEntry->term:%" PRIu64 + ", pSyncNode->pRaftStore->currentTerm:%" PRIu64, + pEntry->term, pSyncNode->pRaftStore->currentTerm); } } @@ -108,7 +107,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { SyncIndex endIndex = newCommitIndex; if (gRaftDetailLog) { - sTrace("syncMaybeAdvanceCommitIndex sync commit %ld", newCommitIndex); + sTrace("syncMaybeAdvanceCommitIndex sync commit %" PRId64, newCommitIndex); } // update commit index diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 2712b4edc6..6dcbb598ae 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -123,8 +123,9 @@ int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, con char host[128]; uint16_t port; syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); - sDebug("vgId:%d, send sync-request-vote to %s:%d, {term:%lu, last-index:%ld, last-term:%lu}", pSyncNode->vgId, host, - port, pMsg->term, pMsg->lastLogTerm, pMsg->lastLogIndex); + sDebug("vgId:%d, send sync-request-vote to %s:%d, {term:%" PRIu64 ", last-index:%" PRId64 ", last-term:%" PRIu64 + "}", + pSyncNode->vgId, host, port, pMsg->term, pMsg->lastLogTerm, pMsg->lastLogIndex); } while (0); SRpcMsg rpcMsg; diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 10f0e0e335..a9c1147fc1 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -134,28 +134,28 @@ char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { // for debug ------------------- void syncIndexMgrPrint(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); - printf("syncIndexMgrPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncIndexMgrPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncIndexMgrPrint2(char *s, SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); - printf("syncIndexMgrPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncIndexMgrPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncIndexMgrLog(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); - sTrace("syncIndexMgrLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncIndexMgrLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncIndexMgrLog2(char *s, SSyncIndexMgr *pObj) { if (gRaftDetailLog) { char *serialized = syncIndexMgr2Str(pObj); - sTrace("syncIndexMgrLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncIndexMgrLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -173,7 +173,7 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, S char host[128]; uint16_t port; syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d index mgr set for %s:%d, term:%lu error", pSyncIndexMgr->pSyncNode->vgId, host, port, term); + sError("vgId:%d index mgr set for %s:%d, term:%" PRIu64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, term); } SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7fa13c9dc8..1db60495c2 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -477,8 +477,8 @@ SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapsho lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i]; } } - sTrace("vgId:%d, sync get snapshot last config index, index:%ld lcindex:%ld", pSyncNode->vgId, snapshotLastApplyIndex, - lastIndex); + sTrace("vgId:%d, sync get snapshot last config index, index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, + snapshotLastApplyIndex, lastIndex); return lastIndex; } @@ -590,7 +590,7 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - sTrace("syncSetQ get pSyncNode is NULL, rid:%ld", rid); + sTrace("syncSetQ get pSyncNode is NULL, rid:%" PRId64, rid); return; } ASSERT(rid == pSyncNode->rid); @@ -602,7 +602,7 @@ void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb) { char* sync2SimpleStr(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid); + sTrace("syncSetRpc get pSyncNode is NULL, rid:%" PRId64, rid); return NULL; } ASSERT(rid == pSyncNode->rid); @@ -819,12 +819,12 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { rpcFreeCont(rpcMsg.pCont); syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); ret = 1; - sDebug("vgId:%d optimized index:%ld success, msgtype:%s,%d", pSyncNode->vgId, retIndex, + sDebug("vgId:%d optimized index:%" PRId64 " success, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType); } else { ret = -1; terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("vgId:%d optimized index:%ld error, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), + sError("vgId:%d optimized index:%" PRId64 " error, msgtype:%s,%d", pSyncNode->vgId, retIndex, TMSG_INFO(pMsg->msgType), pMsg->msgType); } @@ -1391,7 +1391,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache); // life cycle - snprintf(u64buf, sizeof(u64buf), "%ld", pSyncNode->rid); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid); cJSON_AddStringToObject(pRoot, "rid", u64buf); // tla+ server vars @@ -1409,7 +1409,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // tla+ log vars cJSON_AddItemToObject(pRoot, "pLogStore", logStore2Json(pSyncNode->pLogStore)); - snprintf(u64buf, sizeof(u64buf), "%" PRId64 "", pSyncNode->commitIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->commitIndex); cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); // timer ms init @@ -1421,39 +1421,39 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer); cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf); cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClock); cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerLogicClockUser); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimerCB); cJSON_AddStringToObject(pRoot, "FpPingTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->pingTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->pingTimerCounter); cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf); // elect timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer); cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf); cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClock); cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerLogicClockUser); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimerCB); cJSON_AddStringToObject(pRoot, "FpElectTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->electTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->electTimerCounter); cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf); // heartbeat timer snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer); cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf); cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerLogicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClock); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerLogicClockUser); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerLogicClockUser); cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimerCB); cJSON_AddStringToObject(pRoot, "FpHeartbeatTimerCB", u64buf); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", pSyncNode->heartbeatTimerCounter); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSyncNode->heartbeatTimerCounter); cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf); // callback @@ -1527,10 +1527,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 ", lastsnapshot:%" PRId64 ", standby:%d, " "strategy:%d, batch:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, restore:%d, %s", + "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1546,10 +1546,10 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 ", lastsnapshot:%" PRId64 ", standby:%d, " "strategy:%d, batch:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, restore:%d, %s", + "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1590,9 +1590,9 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 ", lastsnapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, restore:%d, %s", + "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1607,9 +1607,9 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 ", lastsnapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, restore:%d, %s", + "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, @@ -1636,9 +1636,9 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); snprintf(s, len, - "vgId:%d, sync %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "vgId:%d, sync %s, term:%" PRIu64 ", commit:%" PRId64 ", beginlog:%" PRId64 ", lastlog:%" PRId64 ", lastsnapshot:%" PRId64 ", standby:%d, " "replica-num:%d, " - "lconfig:%ld, changing:%d, restore:%d", + "lconfig:%" PRId64 ", changing:%d, restore:%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); @@ -1783,7 +1783,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde do { char eventLog[256]; - snprintf(eventLog, sizeof(eventLog), "snapshot sender reset for %lu, newIndex:%d, %s:%d, %p", + snprintf(eventLog, sizeof(eventLog), "snapshot sender reset for: %" PRIu64 ", newIndex:%d, %s:%d, %p", (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); syncNodeEventLog(pSyncNode, eventLog); } while (0); @@ -1839,7 +1839,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde char tmpbuf[512]; char* oldStr = syncCfg2SimpleStr(&oldConfig); char* newStr = syncCfg2SimpleStr(pNewConfig); - snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%ld, %s --> %s", oldConfig.replicaNum, + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum, pNewConfig->replicaNum, lastConfigChangeIndex, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -1863,7 +1863,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde char tmpbuf[512]; char* oldStr = syncCfg2SimpleStr(&oldConfig); char* newStr = syncCfg2SimpleStr(pNewConfig); - snprintf(tmpbuf, sizeof(tmpbuf), "do not config change from %d to %d, index:%ld, %s --> %s", oldConfig.replicaNum, + snprintf(tmpbuf, sizeof(tmpbuf), "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum, pNewConfig->replicaNum, lastConfigChangeIndex, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -1901,7 +1901,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { raftStoreSetTerm(pSyncNode->pRaftStore, term); char tmpBuf[64]; - snprintf(tmpBuf, sizeof(tmpBuf), "update term to %lu", term); + snprintf(tmpBuf, sizeof(tmpBuf), "update term to %" PRIu64, term); syncNodeBecomeFollower(pSyncNode, tmpBuf); raftStoreClearVote(pSyncNode->pRaftStore); } @@ -2189,7 +2189,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%ld", index); + snprintf(logBuf, sizeof(logBuf), "sync node get pre term error, index:%" PRId64, index); syncNodeErrorLog(pSyncNode, logBuf); } while (0); @@ -2206,35 +2206,35 @@ int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex // for debug -------------- void syncNodePrint(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - printf("syncNodePrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncNodePrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncNodePrint2(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - printf("syncNodePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncNodePrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncNodeLog(SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - sTraceLong("syncNodeLog | len:%lu | %s", strlen(serialized), serialized); + sTraceLong("syncNodeLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncNodeLog2(char* s, SSyncNode* pObj) { if (gRaftDetailLog) { char* serialized = syncNode2Str(pObj); - sTraceLong("syncNodeLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("syncNodeLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } void syncNodeLog3(char* s, SSyncNode* pObj) { char* serialized = syncNode2Str(pObj); - sTraceLong("syncNodeLog3 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("syncNodeLog3 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } @@ -2269,7 +2269,7 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } } else { - sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64 "", + sTrace("==syncNodeEqPingTimer== pingTimerLogicClock:%" PRIu64 ", pingTimerLogicClockUser:%" PRIu64, pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } } @@ -2304,7 +2304,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { sError("sync env is stop, syncNodeEqElectTimer"); } } else { - sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64 "", + sTrace("==syncNodeEqElectTimer== electTimerLogicClock:%" PRIu64 ", electTimerLogicClockUser:%" PRIu64, pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser); } } @@ -2399,8 +2399,8 @@ int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) { // log state char logBuf[1024] = {0}; snprintf(logBuf, sizeof(logBuf), - "==syncNodeOnPingCb== vgId:%d, state: %d, %s, term:%lu electTimerLogicClock:%lu, " - "electTimerLogicClockUser:%lu, electTimerMS:%d", + "==syncNodeOnPingCb== vgId:%d, state: %d, %s, term:%" PRIu64 " electTimerLogicClock:%" PRIu64 ", " + "electTimerLogicClockUser:%" PRIu64 ", electTimerMS:%d", ths->vgId, ths->state, syncUtilState2String(ths->state), ths->pRaftStore->currentTerm, ths->electTimerLogicClock, ths->electTimerLogicClockUser, ths->electTimerMS); @@ -2610,7 +2610,7 @@ static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE ASSERT(ret == 0); char eventLog[256]; - snprintf(eventLog, sizeof(eventLog), "maybe leader transfer to %s:%d %lu", + snprintf(eventLog, sizeof(eventLog), "maybe leader transfer to %s:%d %" PRIu64, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, pSyncLeaderTransfer->newLeaderId.addr); syncNodeEventLog(ths, eventLog); @@ -2680,7 +2680,7 @@ static int32_t syncNodeConfigChangeFinish(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyn char tmpbuf[512]; char* oldStr = syncCfg2SimpleStr(&(pFinish->oldCfg)); char* newStr = syncCfg2SimpleStr(&(pFinish->newCfg)); - snprintf(tmpbuf, sizeof(tmpbuf), "config change finish from %d to %d, index:%ld, %s --> %s", + snprintf(tmpbuf, sizeof(tmpbuf), "config change finish from %d to %d, index:%" PRId64 ", %s --> %s", pFinish->oldCfg.replicaNum, pFinish->newCfg.replicaNum, pFinish->newCfgIndex, oldStr, newStr); taosMemoryFree(oldStr); taosMemoryFree(newStr); @@ -2741,7 +2741,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ESyncState state = flag; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "commit by wal from index:%ld to index:%ld", beginIndex, endIndex); + snprintf(eventLog, sizeof(eventLog), "commit by wal from index:%" PRId64 " to index:%" PRId64, beginIndex, endIndex); syncNodeEventLog(ths, eventLog); // execute fsm @@ -2765,7 +2765,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "index:%ld, internalExecute:%d", i, internalExecute); + snprintf(logBuf, sizeof(logBuf), "index:%" PRId64 ", internalExecute:%d", i, internalExecute); syncNodeEventLog(ths, logBuf); } while (0); @@ -2822,7 +2822,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ths->restoreFinish = true; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "restore finish, index:%ld", pEntry->index); + snprintf(eventLog, sizeof(eventLog), "restore finish, index:%" PRId64, pEntry->index); syncNodeEventLog(ths, eventLog); } } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 697c25f304..42a3290d5b 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -133,28 +133,28 @@ char* syncRpcMsg2Str(SRpcMsg* pRpcMsg) { // for debug ---------------------- void syncRpcMsgPrint(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncRpcMsgPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); - printf("syncRpcMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncRpcMsgPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRpcMsgLog(SRpcMsg* pMsg) { char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncRpcMsgLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncRpcMsgLog2(char* s, SRpcMsg* pMsg) { if (gRaftDetailLog) { char* serialized = syncRpcMsg2Str(pMsg); - sTrace("syncRpcMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncRpcMsgLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -242,7 +242,7 @@ cJSON* syncTimeout2Json(const SyncTimeout* pMsg) { cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON_AddNumberToObject(pRoot, "timeoutType", pMsg->timeoutType); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->logicClock); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->logicClock); cJSON_AddStringToObject(pRoot, "logicClock", u64buf); cJSON_AddNumberToObject(pRoot, "timerMS", pMsg->timerMS); snprintf(u64buf, sizeof(u64buf), "%p", pMsg->data); @@ -271,21 +271,21 @@ void syncTimeoutPrint(const SyncTimeout* pMsg) { void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); - printf("syncTimeoutPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncTimeoutPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncTimeoutLog(const SyncTimeout* pMsg) { char* serialized = syncTimeout2Str(pMsg); - sTrace("syncTimeoutLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncTimeoutLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncTimeoutLog2(char* s, const SyncTimeout* pMsg) { if (gRaftDetailLog) { char* serialized = syncTimeout2Str(pMsg); - sTrace("syncTimeoutLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncTimeoutLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -472,7 +472,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -487,7 +487,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -526,28 +526,28 @@ char* syncPing2Str(const SyncPing* pMsg) { // for debug ---------------------- void syncPingPrint(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); - printf("syncPingPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncPingPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncPingPrint2(char* s, const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); - printf("syncPingPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncPingPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncPingLog(const SyncPing* pMsg) { char* serialized = syncPing2Str(pMsg); - sTrace("syncPingLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncPingLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncPingLog2(char* s, const SyncPing* pMsg) { if (gRaftDetailLog) { char* serialized = syncPing2Str(pMsg); - sTrace("syncPingLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncPingLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -734,7 +734,7 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -749,7 +749,7 @@ cJSON* syncPingReply2Json(const SyncPingReply* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -902,7 +902,7 @@ cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg) { cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->seqNum); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->seqNum); cJSON_AddStringToObject(pRoot, "seqNum", u64buf); cJSON_AddNumberToObject(pRoot, "isWeak", pMsg->isWeak); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); @@ -931,28 +931,28 @@ char* syncClientRequest2Str(const SyncClientRequest* pMsg) { // for debug ---------------------- void syncClientRequestPrint(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncClientRequestPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); - printf("syncClientRequestPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncClientRequestPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncClientRequestLog(const SyncClientRequest* pMsg) { char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncClientRequestLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg) { if (gRaftDetailLog) { char* serialized = syncClientRequest2Str(pMsg); - sTrace("syncClientRequestLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncClientRequestLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1101,28 +1101,28 @@ char* syncClientRequestBatch2Str(const SyncClientRequestBatch* pMsg) { // for debug ---------------------- void syncClientRequestBatchPrint(const SyncClientRequestBatch* pMsg) { char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncClientRequestBatchPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncClientRequestBatchPrint2(char* s, const SyncClientRequestBatch* pMsg) { char* serialized = syncClientRequestBatch2Str(pMsg); - printf("syncClientRequestBatchPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncClientRequestBatchPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncClientRequestBatchLog(const SyncClientRequestBatch* pMsg) { char* serialized = syncClientRequestBatch2Str(pMsg); - sTrace("syncClientRequestBatchLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncClientRequestBatchLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncClientRequestBatchLog2(char* s, const SyncClientRequestBatch* pMsg) { if (gRaftDetailLog) { char* serialized = syncClientRequestBatch2Str(pMsg); - sTraceLong("syncClientRequestBatchLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("syncClientRequestBatchLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1201,7 +1201,7 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -1229,11 +1229,11 @@ cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastLogIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastLogIndex); cJSON_AddStringToObject(pRoot, "lastLogIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastLogTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastLogTerm); cJSON_AddStringToObject(pRoot, "lastLogTerm", u64buf); } @@ -1252,28 +1252,28 @@ char* syncRequestVote2Str(const SyncRequestVote* pMsg) { // for debug ---------------------- void syncRequestVotePrint(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); - printf("syncRequestVotePrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncRequestVotePrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); - printf("syncRequestVotePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncRequestVotePrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRequestVoteLog(const SyncRequestVote* pMsg) { char* serialized = syncRequestVote2Str(pMsg); - sTrace("syncRequestVoteLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncRequestVoteLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg) { if (gRaftDetailLog) { char* serialized = syncRequestVote2Str(pMsg); - sTrace("syncRequestVoteLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncRequestVoteLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1352,7 +1352,7 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -1380,7 +1380,7 @@ cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "vote_granted", pMsg->voteGranted); } @@ -1400,28 +1400,28 @@ char* syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg) { // for debug ---------------------- void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); - printf("syncRequestVoteReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncRequestVoteReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); - printf("syncRequestVoteReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncRequestVoteReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg) { char* serialized = syncRequestVoteReply2Str(pMsg); - sTrace("syncRequestVoteReplyLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncRequestVoteReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg) { if (gRaftDetailLog) { char* serialized = syncRequestVoteReply2Str(pMsg); - sTrace("syncRequestVoteReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncRequestVoteReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1502,7 +1502,7 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -1517,7 +1517,7 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -1531,19 +1531,19 @@ cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); cJSON_AddStringToObject(pRoot, "pre_log_term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->commitIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); @@ -1571,28 +1571,28 @@ char* syncAppendEntries2Str(const SyncAppendEntries* pMsg) { // for debug ---------------------- void syncAppendEntriesPrint(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); - printf("syncAppendEntriesPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncAppendEntriesPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); - printf("syncAppendEntriesPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncAppendEntriesPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesLog(const SyncAppendEntries* pMsg) { char* serialized = syncAppendEntries2Str(pMsg); - sTrace("syncAppendEntriesLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncAppendEntriesLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg) { if (gRaftDetailLog) { char* serialized = syncAppendEntries2Str(pMsg); - sTrace("syncAppendEntriesLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncAppendEntriesLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1714,7 +1714,7 @@ cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -1729,7 +1729,7 @@ cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -1743,19 +1743,19 @@ cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->prevLogIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->prevLogIndex); cJSON_AddStringToObject(pRoot, "prevLogIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->prevLogTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->prevLogTerm); cJSON_AddStringToObject(pRoot, "prevLogTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->commitIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); cJSON_AddNumberToObject(pRoot, "dataCount", pMsg->dataCount); @@ -1810,28 +1810,28 @@ char* syncAppendEntriesBatch2Str(const SyncAppendEntriesBatch* pMsg) { // for debug ---------------------- void syncAppendEntriesBatchPrint(const SyncAppendEntriesBatch* pMsg) { char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncAppendEntriesBatchPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesBatchPrint2(char* s, const SyncAppendEntriesBatch* pMsg) { char* serialized = syncAppendEntriesBatch2Str(pMsg); - printf("syncAppendEntriesBatchPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncAppendEntriesBatchPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesBatchLog(const SyncAppendEntriesBatch* pMsg) { char* serialized = syncAppendEntriesBatch2Str(pMsg); - sTrace("syncAppendEntriesBatchLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncAppendEntriesBatchLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncAppendEntriesBatchLog2(char* s, const SyncAppendEntriesBatch* pMsg) { if (gRaftDetailLog) { char* serialized = syncAppendEntriesBatch2Str(pMsg); - sTraceLong("syncAppendEntriesBatchLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("syncAppendEntriesBatchLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -1910,7 +1910,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -1925,7 +1925,7 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -1939,13 +1939,13 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "success", pMsg->success); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->matchIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->matchIndex); cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); } @@ -1964,28 +1964,28 @@ char* syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg) { // for debug ---------------------- void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); - printf("syncAppendEntriesReplyPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncAppendEntriesReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); - printf("syncAppendEntriesReplyPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncAppendEntriesReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg) { char* serialized = syncAppendEntriesReply2Str(pMsg); - sTrace("syncAppendEntriesReplyLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncAppendEntriesReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { if (gRaftDetailLog) { char* serialized = syncAppendEntriesReply2Str(pMsg); - sTrace("syncAppendEntriesReplyLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncAppendEntriesReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -2083,13 +2083,13 @@ cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON_AddNumberToObject(pRoot, "originalRpcType", pMsg->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->fsmMeta.index); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->fsmMeta.index); cJSON_AddStringToObject(pRoot, "fsmMeta.index", u64buf); cJSON_AddNumberToObject(pRoot, "fsmMeta.isWeak", pMsg->fsmMeta.isWeak); cJSON_AddNumberToObject(pRoot, "fsmMeta.code", pMsg->fsmMeta.code); cJSON_AddNumberToObject(pRoot, "fsmMeta.state", pMsg->fsmMeta.state); cJSON_AddStringToObject(pRoot, "fsmMeta.state.str", syncUtilState2String(pMsg->fsmMeta.state)); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->fsmMeta.seqNum); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->fsmMeta.seqNum); cJSON_AddStringToObject(pRoot, "fsmMeta.seqNum", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pMsg->dataLen); @@ -2117,28 +2117,28 @@ char* syncApplyMsg2Str(const SyncApplyMsg* pMsg) { // for debug ---------------------- void syncApplyMsgPrint(const SyncApplyMsg* pMsg) { char* serialized = syncApplyMsg2Str(pMsg); - printf("syncApplyMsgPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncApplyMsgPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg) { char* serialized = syncApplyMsg2Str(pMsg); - printf("syncApplyMsgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncApplyMsgPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncApplyMsgLog(const SyncApplyMsg* pMsg) { char* serialized = syncApplyMsg2Str(pMsg); - sTrace("ssyncApplyMsgLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("ssyncApplyMsgLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg) { if (gRaftDetailLog) { char* serialized = syncApplyMsg2Str(pMsg); - sTrace("syncApplyMsgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncApplyMsgLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -2219,7 +2219,7 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -2234,7 +2234,7 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -2248,23 +2248,23 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->beginIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastConfigIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex); cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf); cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig))); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); cJSON_AddNumberToObject(pRoot, "seq", pMsg->seq); @@ -2294,28 +2294,28 @@ char* syncSnapshotSend2Str(const SyncSnapshotSend* pMsg) { // for debug ---------------------- void syncSnapshotSendPrint(const SyncSnapshotSend* pMsg) { char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncSnapshotSendPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncSnapshotSendPrint2(char* s, const SyncSnapshotSend* pMsg) { char* serialized = syncSnapshotSend2Str(pMsg); - printf("syncSnapshotSendPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncSnapshotSendPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncSnapshotSendLog(const SyncSnapshotSend* pMsg) { char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncSnapshotSendLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncSnapshotSendLog2(char* s, const SyncSnapshotSend* pMsg) { if (gRaftDetailLog) { char* serialized = syncSnapshotSend2Str(pMsg); - sTrace("syncSnapshotSendLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncSnapshotSendLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -2394,7 +2394,7 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -2409,7 +2409,7 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -2423,16 +2423,16 @@ cJSON* syncSnapshotRsp2Json(const SyncSnapshotRsp* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->lastIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastIndex); cJSON_AddStringToObject(pRoot, "lastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->lastTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm); cJSON_AddStringToObject(pRoot, "lastTerm", u64buf); cJSON_AddNumberToObject(pRoot, "ack", pMsg->ack); @@ -2454,28 +2454,28 @@ char* syncSnapshotRsp2Str(const SyncSnapshotRsp* pMsg) { // for debug ---------------------- void syncSnapshotRspPrint(const SyncSnapshotRsp* pMsg) { char* serialized = syncSnapshotRsp2Str(pMsg); - printf("syncSnapshotRspPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncSnapshotRspPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncSnapshotRspPrint2(char* s, const SyncSnapshotRsp* pMsg) { char* serialized = syncSnapshotRsp2Str(pMsg); - printf("syncSnapshotRspPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncSnapshotRspPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncSnapshotRspLog(const SyncSnapshotRsp* pMsg) { char* serialized = syncSnapshotRsp2Str(pMsg); - sTrace("syncSnapshotRspLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncSnapshotRspLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncSnapshotRspLog2(char* s, const SyncSnapshotRsp* pMsg) { if (gRaftDetailLog) { char* serialized = syncSnapshotRsp2Str(pMsg); - sTrace("syncSnapshotRspLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncSnapshotRspLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -2555,7 +2555,7 @@ cJSON* syncLeaderTransfer2Json(const SyncLeaderTransfer* pMsg) { /* cJSON* pSrcId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->srcId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); cJSON_AddStringToObject(pSrcId, "addr", u64buf); { uint64_t u64 = pMsg->srcId.addr; @@ -2570,7 +2570,7 @@ cJSON* syncLeaderTransfer2Json(const SyncLeaderTransfer* pMsg) { cJSON_AddItemToObject(pRoot, "srcId", pSrcId); cJSON* pDestId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->destId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); cJSON_AddStringToObject(pDestId, "addr", u64buf); { uint64_t u64 = pMsg->destId.addr; @@ -2586,7 +2586,7 @@ cJSON* syncLeaderTransfer2Json(const SyncLeaderTransfer* pMsg) { */ cJSON* pNewerId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->newLeaderId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->newLeaderId.addr); cJSON_AddStringToObject(pNewerId, "addr", u64buf); { uint64_t u64 = pMsg->newLeaderId.addr; @@ -2616,28 +2616,28 @@ char* syncLeaderTransfer2Str(const SyncLeaderTransfer* pMsg) { // for debug ---------------------- void syncLeaderTransferPrint(const SyncLeaderTransfer* pMsg) { char* serialized = syncLeaderTransfer2Str(pMsg); - printf("syncLeaderTransferPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncLeaderTransferPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncLeaderTransferPrint2(char* s, const SyncLeaderTransfer* pMsg) { char* serialized = syncLeaderTransfer2Str(pMsg); - printf("syncLeaderTransferPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncLeaderTransferPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncLeaderTransferLog(const SyncLeaderTransfer* pMsg) { char* serialized = syncLeaderTransfer2Str(pMsg); - sTrace("syncLeaderTransferLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncLeaderTransferLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncLeaderTransferLog2(char* s, const SyncLeaderTransfer* pMsg) { if (gRaftDetailLog) { char* serialized = syncLeaderTransfer2Str(pMsg); - sTrace("syncLeaderTransferLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncLeaderTransferLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -2720,13 +2720,13 @@ cJSON* syncReconfigFinish2Json(const SyncReconfigFinish* pMsg) { cJSON_AddItemToObject(pRoot, "oldCfg", pOldCfg); cJSON_AddItemToObject(pRoot, "newCfg", pNewCfg); - snprintf(u64buf, sizeof(u64buf), "%ld", pMsg->newCfgIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->newCfgIndex); cJSON_AddStringToObject(pRoot, "newCfgIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->newCfgTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->newCfgTerm); cJSON_AddStringToObject(pRoot, "newCfgTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pMsg->newCfgSeqNum); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->newCfgSeqNum); cJSON_AddStringToObject(pRoot, "newCfgSeqNum", u64buf); } @@ -2745,28 +2745,28 @@ char* syncReconfigFinish2Str(const SyncReconfigFinish* pMsg) { // for debug ---------------------- void syncReconfigFinishPrint(const SyncReconfigFinish* pMsg) { char* serialized = syncReconfigFinish2Str(pMsg); - printf("syncReconfigFinishPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncReconfigFinishPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncReconfigFinishPrint2(char* s, const SyncReconfigFinish* pMsg) { char* serialized = syncReconfigFinish2Str(pMsg); - printf("syncReconfigFinishPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncReconfigFinishPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncReconfigFinishLog(const SyncReconfigFinish* pMsg) { char* serialized = syncReconfigFinish2Str(pMsg); - sTrace("syncReconfigFinishLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncReconfigFinishLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg) { if (gRaftDetailLog) { char* serialized = syncReconfigFinish2Str(pMsg); - sTrace("syncReconfigFinishLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncReconfigFinishLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 6c381f6e7d..c06bd2338d 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -186,14 +186,14 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { cJSON_AddNumberToObject(pRoot, "batchSize", pRaftCfg->batchSize); char buf64[128]; - snprintf(buf64, sizeof(buf64), "%ld", pRaftCfg->lastConfigIndex); + snprintf(buf64, sizeof(buf64), "%" PRId64, pRaftCfg->lastConfigIndex); cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64); cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfg->configIndexCount); cJSON *pIndexArr = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr); for (int i = 0; i < pRaftCfg->configIndexCount; ++i) { - snprintf(buf64, sizeof(buf64), "%ld", (pRaftCfg->configIndexArr)[i]); + snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfg->configIndexArr)[i]); cJSON *pIndexObj = cJSON_CreateObject(); cJSON_AddStringToObject(pIndexObj, "index", buf64); cJSON_AddItemToArray(pIndexArr, pIndexObj); @@ -306,58 +306,58 @@ int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) { // for debug ---------------------- void syncCfgPrint(SSyncCfg *pCfg) { char *serialized = syncCfg2Str(pCfg); - printf("syncCfgPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("syncCfgPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void syncCfgPrint2(char *s, SSyncCfg *pCfg) { char *serialized = syncCfg2Str(pCfg); - printf("syncCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("syncCfgPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void syncCfgLog(SSyncCfg *pCfg) { char *serialized = syncCfg2Str(pCfg); - sTrace("syncCfgLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("syncCfgLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void syncCfgLog2(char *s, SSyncCfg *pCfg) { char *serialized = syncCfg2Str(pCfg); - sTrace("syncCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncCfgLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } void syncCfgLog3(char *s, SSyncCfg *pCfg) { char *serialized = syncCfg2SimpleStr(pCfg); - sTrace("syncCfgLog3 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("syncCfgLog3 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } void raftCfgPrint(SRaftCfg *pCfg) { char *serialized = raftCfg2Str(pCfg); - printf("raftCfgPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("raftCfgPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void raftCfgPrint2(char *s, SRaftCfg *pCfg) { char *serialized = raftCfg2Str(pCfg); - printf("raftCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("raftCfgPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void raftCfgLog(SRaftCfg *pCfg) { char *serialized = raftCfg2Str(pCfg); - sTrace("raftCfgLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("raftCfgLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void raftCfgLog2(char *s, SRaftCfg *pCfg) { char *serialized = raftCfg2Str(pCfg); - sTrace("raftCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("raftCfgLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 0ee4684ea8..465584a40f 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -130,12 +130,12 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) { cJSON_AddNumberToObject(pRoot, "bytes", pEntry->bytes); cJSON_AddNumberToObject(pRoot, "msgType", pEntry->msgType); cJSON_AddNumberToObject(pRoot, "originalRpcType", pEntry->originalRpcType); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->seqNum); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->seqNum); cJSON_AddStringToObject(pRoot, "seqNum", u64buf); cJSON_AddNumberToObject(pRoot, "isWeak", pEntry->isWeak); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pEntry->index); cJSON_AddStringToObject(pRoot, "index", u64buf); cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen); @@ -246,7 +246,7 @@ int32_t raftCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%ld, bytes:%d", + snprintf(eventLog, sizeof(eventLog), "raft cache add, type:%s,%d, type2:%s,%d, index:%" PRId64 ", bytes:%d", TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, pEntry->index, pEntry->bytes); syncNodeEventLog(pCache->pSyncNode, eventLog); @@ -274,7 +274,7 @@ int32_t raftCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSync do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%" PRId64, TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), (*ppEntry)->originalRpcType, (*ppEntry)->index); syncNodeEventLog(pCache->pSyncNode, eventLog); @@ -306,7 +306,7 @@ int32_t raftCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyn do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%ld", + snprintf(eventLog, sizeof(eventLog), "raft cache get, type:%s,%d, type2:%s,%d, index:%" PRId64, TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), (*ppEntry)->originalRpcType, (*ppEntry)->index); syncNodeEventLog(pCache->pSyncNode, eventLog); @@ -344,7 +344,7 @@ int32_t raftCacheGetAndDel(struct SRaftEntryCache* pCache, SyncIndex index, SSyn do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%ld", + snprintf(eventLog, sizeof(eventLog), "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%" PRId64, TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), (*ppEntry)->originalRpcType, (*ppEntry)->index); syncNodeEventLog(pCache->pSyncNode, eventLog); @@ -415,28 +415,28 @@ char* raftCache2Str(SRaftEntryCache* pCache) { void raftCachePrint(SRaftEntryCache* pCache) { char* serialized = raftCache2Str(pCache); - printf("raftCachePrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("raftCachePrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void raftCachePrint2(char* s, SRaftEntryCache* pCache) { char* serialized = raftCache2Str(pCache); - printf("raftCachePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("raftCachePrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void raftCacheLog(SRaftEntryCache* pCache) { char* serialized = raftCache2Str(pCache); - sTrace("raftCacheLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("raftCacheLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void raftCacheLog2(char* s, SRaftEntryCache* pCache) { if (gRaftDetailLog) { char* serialized = raftCache2Str(pCache); - sTraceLong("raftCacheLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("raftCacheLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } \ No newline at end of file diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 3a44933eea..a135002f44 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -122,7 +122,7 @@ static int32_t raftLogRestoreFromSnapshot(struct SSyncLogStore* pLogStore, SyncI char logBuf[128]; snprintf(logBuf, sizeof(logBuf), - "wal restore from snapshot error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, err, + "wal restore from snapshot error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", snapshotIndex, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); @@ -207,7 +207,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr SyncIndex writeIndex = raftLogWriteIndex(pLogStore); if (pEntry->index != writeIndex) { - sError("vgId:%d wal write index error, entry-index:%ld update to %ld", pData->pSyncNode->vgId, pEntry->index, + sError("vgId:%d wal write index error, entry-index:%" PRId64 " update to %" PRId64, pData->pSyncNode->vgId, pEntry->index, writeIndex); pEntry->index = writeIndex; } @@ -225,7 +225,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr const char* sysErrStr = strerror(errno); char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); @@ -236,7 +236,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr do { char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, + snprintf(eventLog, sizeof(eventLog), "write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); syncNodeEventLog(pData->pSyncNode, eventLog); } while (0); @@ -272,7 +272,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { syncNodeEventLog(pData->pSyncNode, logBuf); @@ -321,7 +321,7 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal truncate error, from-index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + sError("vgId:%d wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); ASSERT(0); @@ -330,7 +330,7 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn // event log do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex); + snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%" PRId64, fromIndex); syncNodeEventLog(pData->pSyncNode, logBuf); } while (0); @@ -381,7 +381,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { const char* sysErrStr = strerror(errno); char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + snprintf(logBuf, sizeof(logBuf), "wal write error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pEntry->index, err, err, errStr, sysErr, sysErrStr); syncNodeErrorLog(pData->pSyncNode, logBuf); @@ -391,7 +391,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { // walFsync(pWal, true); char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "old write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, + snprintf(eventLog, sizeof(eventLog), "old write index:%" PRId64 ", type:%s,%d, type2:%s,%d", pEntry->index, TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); syncNodeEventLog(pData->pSyncNode, eventLog); @@ -418,7 +418,7 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, + snprintf(logBuf, sizeof(logBuf), "wal read error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", index, err, err, errStr, sysErr, sysErrStr); if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { syncNodeEventLog(pData->pSyncNode, logBuf); @@ -466,7 +466,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal truncate error, from-index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + sError("vgId:%d wal truncate error, from-index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); ASSERT(0); @@ -475,7 +475,7 @@ int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { // event log do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%ld", fromIndex); + snprintf(logBuf, sizeof(logBuf), "wal truncate, from-index:%" PRId64, fromIndex); syncNodeEventLog(pData->pSyncNode, logBuf); } while (0); @@ -509,7 +509,7 @@ int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { const char* errStr = tstrerror(err); int32_t sysErr = errno; const char* sysErrStr = strerror(errno); - sError("vgId:%d wal update commit index error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + sError("vgId:%d wal update commit index error, index:%" PRId64 ", err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); ASSERT(0); @@ -546,25 +546,25 @@ cJSON* logStore2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pWal", u64buf); SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); cJSON_AddStringToObject(pRoot, "endIndex", u64buf); int32_t count = raftLogEntryCount(pLogStore); cJSON_AddNumberToObject(pRoot, "entryCount", count); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); cJSON* pEntries = cJSON_CreateArray(); @@ -603,25 +603,25 @@ cJSON* logStoreSimple2Json(SSyncLogStore* pLogStore) { cJSON_AddStringToObject(pRoot, "pWal", u64buf); SyncIndex beginIndex = raftLogBeginIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%ld", beginIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, beginIndex); cJSON_AddStringToObject(pRoot, "beginIndex", u64buf); SyncIndex endIndex = raftLogEndIndex(pLogStore); - snprintf(u64buf, sizeof(u64buf), "%ld", endIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, endIndex); cJSON_AddStringToObject(pRoot, "endIndex", u64buf); int32_t count = raftLogEntryCount(pLogStore); cJSON_AddNumberToObject(pRoot, "entryCount", count); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogWriteIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogWriteIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "WriteIndex", u64buf); snprintf(u64buf, sizeof(u64buf), "%d", raftLogIsEmpty(pLogStore)); cJSON_AddStringToObject(pRoot, "IsEmpty", u64buf); - snprintf(u64buf, sizeof(u64buf), "%ld", raftLogLastIndex(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, raftLogLastIndex(pLogStore)); cJSON_AddStringToObject(pRoot, "LastIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", raftLogLastTerm(pLogStore)); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, raftLogLastTerm(pLogStore)); cJSON_AddStringToObject(pRoot, "LastTerm", u64buf); } @@ -646,14 +646,14 @@ SyncIndex logStoreFirstIndex(SSyncLogStore* pLogStore) { // for debug ----------------- void logStorePrint(SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); - printf("logStorePrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("logStorePrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void logStorePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStore2Str(pLogStore); - printf("logStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("logStorePrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } @@ -661,7 +661,7 @@ void logStorePrint2(char* s, SSyncLogStore* pLogStore) { void logStoreLog(SSyncLogStore* pLogStore) { if (gRaftDetailLog) { char* serialized = logStore2Str(pLogStore); - sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized); + sTraceLong("logStoreLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } } @@ -669,7 +669,7 @@ void logStoreLog(SSyncLogStore* pLogStore) { void logStoreLog2(char* s, SSyncLogStore* pLogStore) { if (gRaftDetailLog) { char* serialized = logStore2Str(pLogStore); - sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTraceLong("logStoreLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } @@ -677,28 +677,28 @@ void logStoreLog2(char* s, SSyncLogStore* pLogStore) { // for debug ----------------- void logStoreSimplePrint(SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("logStoreSimplePrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void logStoreSimplePrint2(char* s, SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); - printf("logStoreSimplePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("logStoreSimplePrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void logStoreSimpleLog(SSyncLogStore* pLogStore) { char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("logStoreSimpleLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void logStoreSimpleLog2(char* s, SSyncLogStore* pLogStore) { if (gRaftDetailLog) { char* serialized = logStoreSimple2Str(pLogStore); - sTrace("logStoreSimpleLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("logStoreSimpleLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index f1c093fe37..18e94e0523 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -135,7 +135,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { SyncIndex newNextIndex = syncNodeGetLastIndex(pSyncNode) + 1; syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, newNextIndex); syncIndexMgrSetIndex(pSyncNode->pMatchIndex, pDestId, SYNC_INDEX_INVALID); - sError("vgId:%d sync get pre term error, nextIndex:%ld, update next-index:%ld, match-index:%d, raftid:%ld", + sError("vgId:%d sync get pre term error, nextIndex:%" PRId64 ", update next-index:%" PRId64 ", match-index:%d, raftid:%" PRId64, pSyncNode->vgId, nextIndex, newNextIndex, SYNC_INDEX_INVALID, pDestId->addr); return -1; @@ -224,7 +224,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { SyncIndex newNextIndex = syncNodeGetLastIndex(pSyncNode) + 1; syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, newNextIndex); syncIndexMgrSetIndex(pSyncNode->pMatchIndex, pDestId, SYNC_INDEX_INVALID); - sError("vgId:%d sync get pre term error, nextIndex:%ld, update next-index:%ld, match-index:%d, raftid:%ld", + sError("vgId:%d sync get pre term error, nextIndex:%" PRId64 ", update next-index:%" PRId64 ", match-index:%d, raftid:%" PRId64, pSyncNode->vgId, nextIndex, newNextIndex, SYNC_INDEX_INVALID, pDestId->addr); return -1; @@ -315,7 +315,7 @@ int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, c uint16_t port; syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); sDebug( - "vgId:%d, send sync-append-entries to %s:%d, {term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, commit:%ld, " + "vgId:%d, send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 ", pterm:%" PRIu64 ", commit:%" PRId64 ", " "datalen:%d}", pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen); @@ -333,12 +333,10 @@ int32_t syncNodeAppendEntriesBatch(SSyncNode* pSyncNode, const SRaftId* destRaft char host[128]; uint16_t port; syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port); - sDebug( - "vgId:%d, send sync-append-entries-batch to %s:%d, {term:%lu, pre-index:%ld, pre-term:%lu, pterm:%lu, " - "commit:%ld, " - "datalen:%d, dataCount:%d}", - pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, - pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount); + sDebug("vgId:%d, send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 + ", pre-term:%" PRIu64 ", pterm:%" PRIu64 ", commit:%" PRId64 ", datalen:%d, datacount:%d}", + pSyncNode->vgId, host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, + pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount); } while (0); SRpcMsg rpcMsg; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index d272e0175f..db1a33c28b 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -55,7 +55,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + "recv sync-request-vote from %s:%d, term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 ", maybe replica already dropped", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); syncNodeEventLog(ths, logBuf); } while (0); @@ -97,7 +97,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + "recv sync-request-vote from %s:%d, term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 ", reply-grant:%d", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); syncNodeEventLog(ths, logBuf); } while (0); @@ -115,7 +115,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%" PRIu64, ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); if (pMsg->term > ths->pRaftStore->currentTerm) { @@ -181,7 +181,7 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + "recv sync-request-vote from %s:%d, term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 ", maybe replica already dropped", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); syncNodeEventLog(ths, logBuf); } while (0); @@ -221,7 +221,7 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + "recv sync-request-vote from %s:%d, term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 ", reply-grant:%d", host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); syncNodeEventLog(ths, logBuf); } while (0); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 4c205d16ec..12af7cf531 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -42,7 +42,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%" PRIu64, ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); // if already drop replica, do not process @@ -53,7 +53,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%" PRIu64 " current_term:%" PRIu64, pMsg->term, ths->pRaftStore->currentTerm); return ret; } @@ -66,7 +66,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%" PRIu64 " current:%" PRIu64, pMsg->term, ths->pRaftStore->currentTerm); syncNodePrint2(logBuf, ths); sError("%s", logBuf); @@ -107,7 +107,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) int32_t ret = 0; char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%" PRIu64, ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); if (pMsg->term < ths->pRaftStore->currentTerm) { @@ -124,7 +124,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%" PRIu64 " current:%" PRIu64, pMsg->term, ths->pRaftStore->currentTerm); syncNodePrint2(logBuf, ths); sError("%s", logBuf); @@ -166,7 +166,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl // print log char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%lu", ths->pRaftStore->currentTerm); + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, term:%" PRIu64, ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); // if already drop replica, do not process @@ -177,7 +177,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl // drop stale response if (pMsg->term < ths->pRaftStore->currentTerm) { - sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%" PRIu64 " current_term:%" PRIu64, pMsg->term, ths->pRaftStore->currentTerm); return ret; } @@ -190,7 +190,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl if (pMsg->term > ths->pRaftStore->currentTerm) { char logBuf[128] = {0}; - snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, error term, receive_term:%lu current_term:%lu", + snprintf(logBuf, sizeof(logBuf), "recv SyncRequestVoteReply, error term, receive_term:%" PRIu64 " current_term:%" PRIu64, pMsg->term, ths->pRaftStore->currentTerm); syncNodePrint2(logBuf, ths); sError("%s", logBuf); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 8dd1349edb..eaeadd3991 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -50,7 +50,7 @@ int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr add, type:%s,%d, seq:%lu, handle:%p, ahandle:%p", + snprintf(eventLog, sizeof(eventLog), "resp mgr add, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, keyCode, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); @@ -77,7 +77,7 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) { SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr get, type:%s,%d, seq:%lu, handle:%p, ahandle:%p", + snprintf(eventLog, sizeof(eventLog), "resp mgr get, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); @@ -98,7 +98,7 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStu SSyncNode *pSyncNode = pObj->data; char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "resp mgr get-and-del, type:%s,%d, seq:%lu, handle:%p, ahandle:%p", + snprintf(eventLog, sizeof(eventLog), "resp mgr get-and-del, type:%s,%d, seq:%" PRIu64 ", handle:%p, ahandle:%p", TMSG_INFO(pStub->rpcMsg.msgType), pStub->rpcMsg.msgType, index, pStub->rpcMsg.info.handle, pStub->rpcMsg.info.ahandle); syncNodeEventLog(pSyncNode, eventLog); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a33f66733b..3079aa17ca 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -153,7 +153,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshotParam snapsho // event log do { char logBuf[128]; - snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to %ld", oldLastConfigIndex, + snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %" PRId64 " to %" PRId64, oldLastConfigIndex, newLastConfigIndex); char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); syncNodeEventLog(pSender->pSyncNode, eventLog); @@ -350,19 +350,19 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { } cJSON *pSnapshot = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyIndex); cJSON_AddStringToObject(pSnapshot, "lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->snapshot.lastApplyTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->snapshot.lastApplyTerm); cJSON_AddStringToObject(pSnapshot, "lastApplyTerm", u64buf); cJSON_AddItemToObject(pRoot, "snapshot", pSnapshot); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->sendingMS); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->sendingMS); cJSON_AddStringToObject(pRoot, "sendingMS", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pSender->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); cJSON_AddNumberToObject(pRoot, "replicaIndex", pSender->replicaIndex); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pSender->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pSender->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); cJSON_AddNumberToObject(pRoot, "finish", pSender->finish); } @@ -389,7 +389,7 @@ char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { syncUtilU642Addr(destId.addr, host, sizeof(host), &port); snprintf(s, len, - "%s {%p s-param:%ld e-param:%ld laindex:%ld laterm:%lu lcindex:%ld seq:%d ack:%d finish:%d pterm:%lu " + "%s {%p s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64 " seq:%d ack:%d finish:%d pterm:%" PRIu64 " " "replica-index:%d %s:%d}", event, pSender, pSender->snapshotParam.start, pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, @@ -640,7 +640,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); cJSON *pFromId = cJSON_CreateObject(); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->fromId.addr); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->fromId.addr); cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; @@ -654,19 +654,19 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddNumberToObject(pFromId, "vgId", pReceiver->fromId.vgId); cJSON_AddItemToObject(pRoot, "fromId", pFromId); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->snapshot.lastApplyIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyIndex); cJSON_AddStringToObject(pRoot, "snapshot.lastApplyIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->snapshot.lastApplyTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastApplyTerm); cJSON_AddStringToObject(pRoot, "snapshot.lastApplyTerm", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->snapshot.lastConfigIndex); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->snapshot.lastConfigIndex); cJSON_AddStringToObject(pRoot, "snapshot.lastConfigIndex", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->term); cJSON_AddStringToObject(pRoot, "term", u64buf); - snprintf(u64buf, sizeof(u64buf), "%lu", pReceiver->privateTerm); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pReceiver->privateTerm); cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); } @@ -692,8 +692,8 @@ char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) syncUtilU642Addr(fromId.addr, host, sizeof(host), &port); snprintf(s, len, - "%s {%p start:%d ack:%d term:%lu pterm:%lu from:%s:%d s-param:%ld e-param:%ld laindex:%ld laterm:%lu " - "lcindex:%ld}", + "%s {%p start:%d ack:%d term:%" PRIu64 " pterm:%" PRIu64 " from:%s:%d s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " " + "lcindex:%" PRId64 "}", event, pReceiver, pReceiver->start, pReceiver->ack, pReceiver->term, pReceiver->privateTerm, host, port, pReceiver->snapshotParam.start, pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm, pReceiver->snapshot.lastConfigIndex); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 2c43312064..1d46d71a05 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -109,7 +109,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted); cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes); - snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesGranted->term); cJSON_AddStringToObject(pRoot, "term", u64buf); cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum); cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader); @@ -135,27 +135,27 @@ char *voteGranted2Str(SVotesGranted *pVotesGranted) { // for debug ------------------- void voteGrantedPrint(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); - printf("voteGrantedPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("voteGrantedPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void voteGrantedPrint2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); - printf("voteGrantedPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("voteGrantedPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void voteGrantedLog(SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); - sTrace("voteGrantedLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("voteGrantedLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void voteGrantedLog2(char *s, SVotesGranted *pObj) { char *serialized = voteGranted2Str(pObj); - sTrace("voteGrantedLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("voteGrantedLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } @@ -243,7 +243,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond); cJSON_AddNumberToObject(pRoot, "respondNum", respondNum); - snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term); cJSON_AddStringToObject(pRoot, "term", u64buf); snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode); cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf); @@ -264,26 +264,26 @@ char *votesRespond2Str(SVotesRespond *pVotesRespond) { // for debug ------------------- void votesRespondPrint(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); - printf("votesRespondPrint | len:%lu | %s \n", strlen(serialized), serialized); + printf("votesRespondPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); fflush(NULL); taosMemoryFree(serialized); } void votesRespondPrint2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); - printf("votesRespondPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized); + printf("votesRespondPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); fflush(NULL); taosMemoryFree(serialized); } void votesRespondLog(SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); - sTrace("votesRespondLog | len:%lu | %s", strlen(serialized), serialized); + sTrace("votesRespondLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); taosMemoryFree(serialized); } void votesRespondLog2(char *s, SVotesRespond *pObj) { char *serialized = votesRespond2Str(pObj); - sTrace("votesRespondLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized); + sTrace("votesRespondLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); taosMemoryFree(serialized); } diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 7cd97695f9..de82df3fbd 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -45,19 +45,19 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { if (cbMeta.index > beginIndex) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu \n", + "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag, cbMeta.term); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { - sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -65,7 +65,7 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -147,7 +147,7 @@ int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_ void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { - sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%ld, code:%d, currentTerm:%lu, term:%lu", cbMeta.flag, + sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%" PRId64 ", code:%d, currentTerm:%" PRIu64 ", term:%" PRIu64, cbMeta.flag, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); } @@ -267,7 +267,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { pMsg->msgType = 9999; pMsg->contLen = 256; pMsg->pCont = rpcMallocCont(pMsg->contLen); - snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count, taosGetTimestampMs()); return pMsg; } diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index c96e337378..80a5e65274 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -44,18 +44,18 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { if (cbMeta.index > beginIndex) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { - sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -63,7 +63,7 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s flag:%lu\n", pFsm, + "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s flag:%" PRIu64 "\n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -78,7 +78,7 @@ int32_t GetSnapshotCb(struct SSyncFSM* pFsm, SSnapshot* pSnapshot) { void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFinishCb=="); } void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { - sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%ld, code:%d, currentTerm:%lu, term:%lu", cbMeta.flag, + sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%" PRId64 ", code:%d, currentTerm:%" PRIu64 ", term:%" PRIu64, cbMeta.flag, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term); } @@ -188,7 +188,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { pMsg->msgType = 9999; pMsg->contLen = 256; pMsg->pCont = rpcMallocCont(pMsg->contLen); - snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count, taosGetTimestampMs()); return pMsg; } diff --git a/source/libs/sync/test/syncIndexMgrTest.cpp b/source/libs/sync/test/syncIndexMgrTest.cpp index 0ad69f0f51..23b8693a0b 100644 --- a/source/libs/sync/test/syncIndexMgrTest.cpp +++ b/source/libs/sync/test/syncIndexMgrTest.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]); // SyncTerm term = syncIndexMgrGetTerm(pSyncIndexMgr, &ids[i]); - // printf("%d: index:%ld term:%lu \n", i, idx, term); + // printf("%d: index:%" PRId64 " term:%" PRIu64 " \n", i, idx, term); } printf("---------------------------------------\n"); diff --git a/source/libs/sync/test/syncIndexTest.cpp b/source/libs/sync/test/syncIndexTest.cpp index 07a05d437e..91c7800756 100644 --- a/source/libs/sync/test/syncIndexTest.cpp +++ b/source/libs/sync/test/syncIndexTest.cpp @@ -13,7 +13,7 @@ void print(SHashObj *pNextIndex) { SRaftId *pRaftId = (SRaftId *)key; - printf("key:<%lu, %d>, value:%lu \n", pRaftId->addr, pRaftId->vgId, *p); + printf("key:<" PRIu64 ", %d>, value:%" PRIu64 " \n", pRaftId->addr, pRaftId->vgId, *p); p = (uint64_t *)taosHashIterate(pNextIndex, p); } } diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index 90560e91e7..65da0f6631 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -15,14 +15,14 @@ int main(int argc, char** argv) { char host[128]; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); - printf("%lu -> %s:%d \n", u64, host, port); + printf("" PRIu64 " -> %s:%d \n", u64, host, port); } else if (argc == 3) { uint64_t u64; char* host = argv[1]; uint16_t port = atoi(argv[2]); u64 = syncUtilAddr2U64(host, port); - printf("%s:%d -> %lu \n", host, port, u64); + printf("%s:%d ->: %" PRIu64 " \n", host, port, u64); } else { usage(argv[0]); exit(-1); diff --git a/source/libs/sync/test/syncRaftLogTest.cpp b/source/libs/sync/test/syncRaftLogTest.cpp index 7903e86749..278113919a 100644 --- a/source/libs/sync/test/syncRaftLogTest.cpp +++ b/source/libs/sync/test/syncRaftLogTest.cpp @@ -38,7 +38,7 @@ void test1() { int64_t firstVer = walGetFirstVer(pWal); int64_t lastVer = walGetLastVer(pWal); - printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + printf("firstVer:%" PRId64 " lastVer:%" PRId64 " \n", firstVer, lastVer); walClose(pWal); } @@ -68,7 +68,7 @@ void test2() { int64_t firstVer = walGetFirstVer(pWal); int64_t lastVer = walGetLastVer(pWal); - printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + printf("firstVer:%" PRId64 " lastVer:%" PRId64 " \n", firstVer, lastVer); walClose(pWal); } @@ -92,7 +92,7 @@ void test3() { int64_t firstVer = walGetFirstVer(pWal); int64_t lastVer = walGetLastVer(pWal); - printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + printf("firstVer:%" PRId64 " lastVer:%" PRId64 " \n", firstVer, lastVer); walClose(pWal); } @@ -124,7 +124,7 @@ void test4() { int64_t firstVer = walGetFirstVer(pWal); int64_t lastVer = walGetLastVer(pWal); - printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + printf("firstVer:%" PRId64 " lastVer:%" PRId64 " \n", firstVer, lastVer); walClose(pWal); } @@ -149,7 +149,7 @@ void test5() { int64_t firstVer = walGetFirstVer(pWal); int64_t lastVer = walGetLastVer(pWal); - printf("firstVer:%ld lastVer:%ld \n", firstVer, lastVer); + printf("firstVer:%" PRId64 " lastVer:%" PRId64 " \n", firstVer, lastVer); walClose(pWal); } diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 9e0c2ecc29..0cb9b51fba 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -413,7 +413,7 @@ void test6() { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); bool isEmpty = walIsEmpty(pWal); - printf("before -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + printf("before -------- firstVer:%" PRId64 " lastVer:%" PRId64 " isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); logStoreDestory(pLogStore); @@ -429,7 +429,7 @@ void test6() { SyncIndex firstVer = walGetFirstVer(pWal); SyncIndex lastVer = walGetLastVer(pWal); bool isEmpty = walIsEmpty(pWal); - printf("after -------- firstVer:%ld lastVer:%ld isEmpty:%d \n", firstVer, lastVer, isEmpty); + printf("after -------- firstVer:%" PRId64 " lastVer:%" PRId64 " isEmpty:%d \n", firstVer, lastVer, isEmpty); } while (0); logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); diff --git a/source/libs/sync/test/syncRaftLogTest3.cpp b/source/libs/sync/test/syncRaftLogTest3.cpp index ea1788c545..fd4cade31c 100644 --- a/source/libs/sync/test/syncRaftLogTest3.cpp +++ b/source/libs/sync/test/syncRaftLogTest3.cpp @@ -92,13 +92,13 @@ void test1() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test1"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + sTrace("hasSnapshot:%d, lastApplyIndex:%" PRId64 ", lastApplyTerm:%" PRIu64, hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - sTrace("lastIndex: %ld", lastIndex); - sTrace("lastTerm: %lu", lastTerm); - sTrace("syncStartIndex: %ld", syncStartIndex); - sTrace("%ld's preIndex: %ld", testIndex, preIndex); - sTrace("%ld's preTerm: %lu", testIndex, preTerm); + sTrace("lastIndex: %" PRId64, lastIndex); + sTrace("lastTerm: %" PRIu64, lastTerm); + sTrace("syncStartIndex: %" PRId64, syncStartIndex); + sTrace("" PRId64 "'s preIndex: %" PRId64, testIndex, preIndex); + sTrace("" PRId64 "'s preTerm: %" PRIu64, testIndex, preTerm); if (gAssert) { assert(lastIndex == -1); @@ -154,11 +154,11 @@ void test2() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test2"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + sTrace("hasSnapshot:%d, lastApplyIndex:%" PRId64 ", lastApplyTerm:%" PRIu64, hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - sTrace("lastIndex: %ld", lastIndex); - sTrace("lastTerm: %lu", lastTerm); - sTrace("syncStartIndex: %ld", syncStartIndex); + sTrace("lastIndex: %" PRId64, lastIndex); + sTrace("lastTerm: %" PRIu64, lastTerm); + sTrace("syncStartIndex: %" PRId64, syncStartIndex); if (gAssert) { assert(lastIndex == 10); @@ -170,8 +170,8 @@ void test2() { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); - sTrace("%ld's preIndex: %ld", i, preIndex); - sTrace("%ld's preTerm: %lu", i, preTerm); + sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex); + sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm); if (gAssert) { SyncIndex preIndexArr[12] = {-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -214,13 +214,13 @@ void test3() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test3"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + sTrace("hasSnapshot:%d, lastApplyIndex:%" PRId64 ", lastApplyTerm:%" PRIu64, hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - sTrace("lastIndex: %ld", lastIndex); - sTrace("lastTerm: %lu", lastTerm); - sTrace("syncStartIndex: %ld", syncStartIndex); - sTrace("%d's preIndex: %ld", 6, preIndex); - sTrace("%d's preTerm: %lu", 6, preTerm); + sTrace("lastIndex: %" PRId64, lastIndex); + sTrace("lastTerm: %" PRIu64, lastTerm); + sTrace("syncStartIndex: %" PRId64, syncStartIndex); + sTrace("%d's preIndex: %" PRId64, 6, preIndex); + sTrace("%d's preTerm: %" PRIu64, 6, preTerm); if (gAssert) { assert(lastIndex == 5); @@ -276,11 +276,11 @@ void test4() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test4"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + sTrace("hasSnapshot:%d, lastApplyIndex:%" PRId64 ", lastApplyTerm:%" PRIu64, hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - sTrace("lastIndex: %ld", lastIndex); - sTrace("lastTerm: %lu", lastTerm); - sTrace("syncStartIndex: %ld", syncStartIndex); + sTrace("lastIndex: %" PRId64, lastIndex); + sTrace("lastTerm: %" PRIu64, lastTerm); + sTrace("syncStartIndex: %" PRId64, syncStartIndex); if (gAssert) { assert(lastIndex == 10); @@ -292,8 +292,8 @@ void test4() { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); - sTrace("%ld's preIndex: %ld", i, preIndex); - sTrace("%ld's preTerm: %lu", i, preTerm); + sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex); + sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm); } logStoreDestory(pLogStore); @@ -344,18 +344,18 @@ void test5() { SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); sTrace("test5"); - sTrace("hasSnapshot:%d, lastApplyIndex:%ld, lastApplyTerm:%lu", hasSnapshot, snapshot.lastApplyIndex, + sTrace("hasSnapshot:%d, lastApplyIndex:%" PRId64 ", lastApplyTerm:%" PRIu64, hasSnapshot, snapshot.lastApplyIndex, snapshot.lastApplyTerm); - sTrace("lastIndex: %ld", lastIndex); - sTrace("lastTerm: %lu", lastTerm); - sTrace("syncStartIndex: %ld", syncStartIndex); + sTrace("lastIndex: %" PRId64, lastIndex); + sTrace("lastTerm: %" PRIu64, lastTerm); + sTrace("syncStartIndex: %" PRId64, syncStartIndex); for (SyncIndex i = 11; i >= 6; --i) { SyncIndex preIndex = syncNodeGetPreIndex(pSyncNode, i); SyncTerm preTerm = syncNodeGetPreTerm(pSyncNode, i); - sTrace("%ld's preIndex: %ld", i, preIndex); - sTrace("%ld's preTerm: %lu", i, preTerm); + sTrace("" PRId64 "'s preIndex: %" PRId64, i, preIndex); + sTrace("" PRId64 "'s preTerm: %" PRIu64, i, preTerm); if (gAssert) { SyncIndex preIndexArr[12] = {9999, 9999, 9999, 9999, 9999, 9999, 5, 6, 7, 8, 9, 10}; diff --git a/source/libs/sync/test/syncRefTest.cpp b/source/libs/sync/test/syncRefTest.cpp index 96062b1a91..90923a87ee 100644 --- a/source/libs/sync/test/syncRefTest.cpp +++ b/source/libs/sync/test/syncRefTest.cpp @@ -32,7 +32,7 @@ typedef struct SyncObj { static void syncFreeObj(void *param) { SyncObj *pObj = (SyncObj *)param; - printf("syncFreeObj name:%s rid:%ld \n", pObj->name, pObj->rid); + printf("syncFreeObj name:%s rid:%" PRId64 " \n", pObj->name, pObj->rid); taosMemoryFree(pObj); } @@ -66,7 +66,7 @@ int64_t start() { return -1; } - printf("start name:%s rid:%ld \n", pObj->name, pObj->rid); + printf("start name:%s rid:%" PRId64 " \n", pObj->name, pObj->rid); return pObj->rid; } @@ -74,7 +74,7 @@ void stop(int64_t rid) { SyncObj *pObj = (SyncObj *)taosAcquireRef(tsNodeRefId, rid); if (pObj == NULL) return; - printf("stop name:%s rid:%ld \n", pObj->name, pObj->rid); + printf("stop name:%s rid:%" PRId64 " \n", pObj->name, pObj->rid); pObj->data = NULL; taosReleaseRef(tsNodeRefId, pObj->rid); @@ -89,7 +89,7 @@ void *func(void *param) { SyncObj *pObj = (SyncObj *)taosAcquireRef(tsNodeRefId, rid); if (pObj != NULL) { - printf("taosAcquireRef sleep:%d, name:%s, rid:%ld \n", ms, pObj->name, pObj->rid); + printf("taosAcquireRef sleep:%d, name:%s, rid:%" PRId64 " \n", ms, pObj->name, pObj->rid); } else { printf("taosAcquireRef sleep:%d, NULL! \n", ms); } diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 66f347b620..9148ab6195 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -40,25 +40,25 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { if (cbMeta.index > beginIndex) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } else { - sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); } @@ -143,7 +143,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { pMsg->msgType = 9999; pMsg->contLen = 256; pMsg->pCont = rpcMallocCont(pMsg->contLen); - snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count, taosGetTimestampMs()); return pMsg; } diff --git a/source/libs/sync/test/syncRespMgrTest.cpp b/source/libs/sync/test/syncRespMgrTest.cpp index 93a7ce430f..9e982e0a59 100644 --- a/source/libs/sync/test/syncRespMgrTest.cpp +++ b/source/libs/sync/test/syncRespMgrTest.cpp @@ -23,7 +23,7 @@ void syncRespMgrInsert(uint64_t count) { stub.rpcMsg.info.ahandle = (void *)(200 + i); stub.rpcMsg.info.handle = (void *)(300 + i); uint64_t ret = syncRespMgrAdd(pMgr, &stub); - printf("insert %lu \n", ret); + printf("insert: %" PRIu64 " \n", ret); } } @@ -35,7 +35,7 @@ void syncRespMgrDelTest(uint64_t begin, uint64_t end) { } void printStub(SRespStub *p) { - printf("createTime:%ld, rpcMsg.code:%d rpcMsg.ahandle:%ld rpcMsg.handle:%ld \n", p->createTime, p->rpcMsg.code, + printf("createTime:%" PRId64 ", rpcMsg.code:%d rpcMsg.ahandle:%" PRId64 " rpcMsg.handle:%" PRId64 " \n", p->createTime, p->rpcMsg.code, (int64_t)(p->rpcMsg.info.ahandle), (int64_t)(p->rpcMsg.info.handle)); } void syncRespMgrPrint() { @@ -52,24 +52,24 @@ void syncRespMgrPrint() { } void syncRespMgrGetTest(uint64_t i) { - printf("------syncRespMgrGetTest------- %lu -- \n", i); + printf("------syncRespMgrGetTest-------: %" PRIu64 " -- \n", i); SRespStub stub; int32_t ret = syncRespMgrGet(pMgr, i, &stub); if (ret == 1) { printStub(&stub); } else if (ret == 0) { - printf("%ld notFound \n", i); + printf("" PRId64 " notFound \n", i); } } void syncRespMgrGetAndDelTest(uint64_t i) { - printf("------syncRespMgrGetAndDelTest-------%lu-- \n", i); + printf("------syncRespMgrGetAndDelTest-------" PRIu64 "-- \n", i); SRespStub stub; int32_t ret = syncRespMgrGetAndDel(pMgr, i, &stub); if (ret == 1) { printStub(&stub); } else if (ret == 0) { - printf("%ld notFound \n", i); + printf("" PRId64 " notFound \n", i); } } diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index 954bdd8ea5..9e50fa62ef 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -43,25 +43,25 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { if (cbMeta.index > beginIndex) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } else { - sTrace("==callback== ==CommitCb== do not apply again %ld", cbMeta.index); + sTrace("==callback== ==CommitCb== do not apply again %" PRId64, cbMeta.index); } } void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } @@ -172,7 +172,7 @@ int main(int argc, char **argv) { if (argc >= 2) { snapshotLastApplyIndex = atoi(argv[1]); } - sTrace("--snapshotLastApplyIndex : %ld \n", snapshotLastApplyIndex); + sTrace("--snapshotLastApplyIndex : %" PRId64 " \n", snapshotLastApplyIndex); int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); assert(ret == 0); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 2c08910aa8..714b73a9e5 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -40,8 +40,8 @@ void cleanup() { walCleanUp(); } void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " - "currentTerm:%lu \n", + "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " " + "currentTerm:%" PRIu64 " \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag, cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); @@ -50,8 +50,8 @@ void CommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " - "currentTerm:%lu \n", + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " " + "currentTerm:%" PRIu64 " \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag, cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); @@ -60,8 +60,8 @@ void PreCommitCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) void RollBackCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " - "currentTerm:%lu \n", + "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " " + "currentTerm:%" PRIu64 " \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag, cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); @@ -128,8 +128,8 @@ int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d, gSnapshotLastApplyIndex:%ld, " - "gSnapshotLastApplyTerm:%ld", + "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d, gSnapshotLastApplyIndex:%" PRId64 ", " + "gSnapshotLastApplyTerm:%" PRId64, pFsm, pWriter, isApply, gSnapshotLastApplyIndex, gSnapshotLastApplyTerm); sTrace("%s", logBuf); @@ -148,7 +148,7 @@ void RestoreFinishCb(struct SSyncFSM* pFsm) { sTrace("==callback== ==RestoreFini void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta) { char* s = syncCfg2Str(&(cbMeta.newCfg)); - sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%ld, code:%d, currentTerm:%lu, term:%lu, newCfg:%s", + sTrace("==callback== ==ReConfigCb== flag:0x%lX, index:%" PRId64 ", code:%d, currentTerm:%" PRIu64 ", term:%" PRIu64 ", newCfg:%s", cbMeta.flag, cbMeta.index, cbMeta.code, cbMeta.currentTerm, cbMeta.term, s); taosMemoryFree(s); } @@ -156,8 +156,8 @@ void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMe void LeaderTransferCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), - "==callback== ==LeaderTransferCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " - "currentTerm:%lu \n", + "==callback== ==LeaderTransferCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s, flag:%" PRIu64 ", term:%" PRIu64 " " + "currentTerm:%" PRIu64 " \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), cbMeta.flag, cbMeta.term, cbMeta.currentTerm); syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); @@ -300,7 +300,7 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { pMsg->msgType = TDMT_VND_SUBMIT; pMsg->contLen = 256; pMsg->pCont = rpcMallocCont(pMsg->contLen); - snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-%ld", myIndex, i, count, taosGetTimestampMs()); + snprintf((char*)(pMsg->pCont), pMsg->contLen, "value-myIndex:%u-%d-%d-" PRId64, myIndex, i, count, taosGetTimestampMs()); return pMsg; } diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index 34c8eb0f56..d99923a8b9 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -33,7 +33,7 @@ const char *pDir = "./syncWriteTest"; void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==CommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } @@ -41,14 +41,14 @@ void CommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { void PreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; snprintf(logBuf, sizeof(logBuf), - "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, + "==callback== ==PreCommitCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } void RollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) { char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", + snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%" PRId64 ", isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state)); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index cdae73bfb9..22229ea0e8 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -258,8 +258,8 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { } static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { - SPage **ppPage; - int h; + SPage **ppPage; + uint32_t h; h = tdbPCachePageHash(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); (*ppPage) && *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index dad4511491..60eaf467ae 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -370,7 +370,7 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage init = 1; nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); - tdbTrace("tdbttl pager:%p, pgno:%d, nRead:%ld", pPager, pgno, nRead); + tdbTrace("tdbttl pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { ASSERT(0); return -1; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index f699df6883..e73ddedd73 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -249,30 +249,30 @@ int transAsyncSend(SAsyncPool* pool, queue* mq); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, id) \ - do { \ - if (id > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(transGetRefMgt(), id); \ - if (exh2 == NULL || id != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, id); \ - goto _return1; \ - } \ - } else if (id == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(transGetRefMgt(), id); \ - if (exh2 == NULL || id == exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - id = exh1->refId; \ - } \ - } else if (id < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ +#define ASYNC_CHECK_HANDLE(exh1, id) \ + do { \ + if (id > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(transGetRefMgt(), id); \ + if (exh2 == NULL || id != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1:%" PRIu64 ", ref2:%" PRIu64, exh1, \ + exh2 ? exh2->refId : 0, id); \ + goto _return1; \ + } \ + } else if (id == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(transGetRefMgt(), id); \ + if (exh2 == NULL || id == exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1:%" PRIu64 ", ref2:%" PRIu64, exh1, id, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + id = exh1->refId; \ + } \ + } else if (id < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ } while (0) int transInitBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48e7d7c91d..e9e439971c 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -58,7 +58,7 @@ void* rpcOpen(const SRpcInit* pInit) { uint32_t ip = 0; if (pInit->connType == TAOS_CONN_SERVER) { if (transValidLocalFqdn(pInit->localFqdn, &ip) != 0) { - tError("invalid fqdn: %s, errmsg: %s", pInit->localFqdn, terrstr()); + tError("invalid fqdn:%s, errmsg:%s", pInit->localFqdn, terrstr()); taosMemoryFree(pRpc); return NULL; } @@ -86,7 +86,7 @@ void rpcClose(void* arg) { tInfo("start to close rpc"); transRemoveExHandle(transGetInstMgt(), (int64_t)arg); transReleaseExHandle(transGetInstMgt(), (int64_t)arg); - tInfo("finish to close rpc"); + tInfo("rpc is closed"); return; } void rpcCloseImpl(void* arg) { @@ -112,7 +112,7 @@ void* rpcMallocCont(int32_t contLen) { void rpcFreeCont(void* cont) { if (cont == NULL) return; taosMemoryFree((char*)cont - TRANS_MSG_OVERHEAD); - tTrace("free mem: %p", (char*)cont - TRANS_MSG_OVERHEAD); + tTrace("free mem:%p", (char*)cont - TRANS_MSG_OVERHEAD); } void* rpcReallocCont(void* ptr, int32_t contLen) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 4b6ff43ba8..0600b1130d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -184,22 +184,22 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - uint64_t ahandle = head->ahandle; \ - CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ - if (T_REF_VAL_GET(conn) > 1) { \ - transUnrefCliHandle(conn); \ - } \ - destroyCmsg(pMsg); \ - cliReleaseUnfinishedMsg(conn); \ - addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ - return; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + uint64_t ahandle = head->ahandle; \ + CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + tDebug("%s conn %p receive release request, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ + if (T_REF_VAL_GET(conn) > 1) { \ + transUnrefCliHandle(conn); \ + } \ + destroyCmsg(pMsg); \ + cliReleaseUnfinishedMsg(conn); \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ + return; \ + } \ } while (0) #define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ @@ -353,7 +353,7 @@ void cliHandleResp(SCliConn* conn) { } STraceId* trace = &transMsg.info.traceId; - tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn), + tGTrace("%s conn %p %s received from %s:%d, local info:%s:%d, msg size:%d, code:0x%x", CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); @@ -573,7 +573,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tWarn("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + tWarn("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); @@ -655,7 +655,7 @@ static void cliSendCb(uv_write_t* req, int status) { if (status == 0) { tTrace("%s conn %p data already was written out", CONN_GET_INST_LABEL(pConn), pConn); } else { - tError("%s conn %p failed to write: %s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); + tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); cliHandleExcept(pConn); return; } @@ -719,7 +719,7 @@ void cliConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; if (status != 0) { - tError("%s conn %p failed to connect server: %s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); + tError("%s conn %p failed to connect server:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); cliHandleExcept(pConn); return; } @@ -852,7 +852,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { tTrace("%s conn %p try to connect to %s:%d", pTransInst->label, conn, conn->ip, conn->port); ret = uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, cliConnCb); if (ret != 0) { - tTrace("%s conn %p failed to connect to %s:%d, reason: %s", pTransInst->label, conn, conn->ip, conn->port, + tTrace("%s conn %p failed to connect to %s:%d, reason:%s", pTransInst->label, conn, conn->ip, conn->port, uv_err_name(ret)); cliHandleExcept(conn); return; @@ -883,7 +883,7 @@ static void cliAsyncCb(uv_async_t* handle) { count++; } if (count >= 2) { - tTrace("cli process batch size: %d", count); + tTrace("cli process batch size:%d", count); } } @@ -910,7 +910,7 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, int err = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread, (void*)(pThrd)); if (err == 0) { - tDebug("success to create tranport-cli thread %d", i); + tDebug("success to create tranport-cli thread:%d", i); } cli->pThreadObj[i] = pThrd; } @@ -1222,7 +1222,7 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transAsyncSend(pThrd->asyncPool, &(cliMsg->q)) == 0); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); @@ -1260,7 +1260,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->refId = (int64_t)shandle; STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); transAsyncSend(pThrd->asyncPool, &(cliMsg->q)); @@ -1294,7 +1294,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { cliMsg->refId = (int64_t)shandle; SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; - tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); + tDebug("%s update epset at thread:%08" PRId64, pTransInst->label, thrd->pid); transAsyncSend(thrd->asyncPool, &(cliMsg->q)); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5f6e3db615..812123441c 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -136,7 +136,7 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { } else { p->cap = p->total; p->buf = taosMemoryRealloc(p->buf, p->cap); - tTrace("internal malloc mem: %p, size: %d", p->buf, p->cap); + tTrace("internal malloc mem:%p, size:%d", p->buf, p->cap); uvBuf->base = p->buf + p->len; uvBuf->len = p->cap - p->len; @@ -221,7 +221,7 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { taosThreadMutexUnlock(&item->mtx); int64_t el = taosGetTimestampUs() - st; if (el > 50) { - // tInfo("lock and unlock cost: %d", (int)el); + // tInfo("lock and unlock cost:%d", (int)el); } return uv_async_send(async); } @@ -446,7 +446,7 @@ int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_ } } - tTrace("timer %p put task into delay queue, timeoutMs: %" PRIu64 "", queue->timer, timeoutMs); + tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0); return 0; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 9e4abf09cf..a239f90c29 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -245,11 +245,11 @@ static void uvHandleReq(SSvrConn* pConn) { if (pConn->status == ConnNormal && pHead->noResp == 0) { transRefSrvHandle(pConn); - tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", transLabel(pTransInst), pConn, + tGTrace("%s conn %p %s received from %s:%d, local info:%s:%d, msg size:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), transMsg.contLen); } else { - tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, resp:%d, code: %d", + tGTrace("%s conn %p %s received from %s:%d, local info:%s:%d, msg size:%d, resp:%d, code:%d", transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), transMsg.contLen, pHead->noResp, transMsg.code); @@ -265,7 +265,7 @@ static void uvHandleReq(SSvrConn* pConn) { transMsg.info.refId = pConn->refId; transMsg.info.traceId = pHead->traceId; - tGTrace("%s handle %p conn: %p translated to app, refId: %" PRIu64 "", transLabel(pTransInst), transMsg.info.handle, + tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, pConn->refId); assert(transMsg.info.handle != NULL); @@ -292,7 +292,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { STrans* pTransInst = conn->pTransInst; if (nread > 0) { pBuf->len += nread; - tTrace("%s conn %p total read: %d, current read: %d", transLabel(pTransInst), conn, pBuf->len, (int)nread); + tTrace("%s conn %p total read:%d, current read:%d", transLabel(pTransInst), conn, pBuf->len, (int)nread); if (transReadComplete(pBuf)) { tTrace("%s conn %p alread read complete packet", transLabel(pTransInst), conn); uvHandleReq(conn); @@ -305,7 +305,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } - tWarn("%s conn %p read error: %s", transLabel(pTransInst), conn, uv_err_name(nread)); + tWarn("%s conn %p read error:%s", transLabel(pTransInst), conn, uv_err_name(nread)); if (nread < 0) { conn->broken = true; if (conn->status == ConnAcquire) { @@ -414,7 +414,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { STrans* pTransInst = pConn->pTransInst; STraceId* trace = &pMsg->info.traceId; - tGTrace("%s conn %p %s is sent to %s:%d, local info: %s:%d, msglen:%d", transLabel(pTransInst), pConn, + tGTrace("%s conn %p %s is sent to %s:%d, local info:%s:%d, msglen:%d", transLabel(pTransInst), pConn, TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port), len); pHead->msgLen = htonl(len); @@ -538,7 +538,7 @@ static void uvAcceptAsyncCb(uv_async_t* async) { static void uvShutDownCb(uv_shutdown_t* req, int status) { if (status != 0) { - tDebug("conn failed to shut down: %s", uv_err_name(status)); + tDebug("conn failed to shut down:%s", uv_err_name(status)); } uv_close((uv_handle_t*)req->handle, uvDestroyConn); taosMemoryFree(req); @@ -599,7 +599,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { tError("read error %s", uv_err_name(nread)); } // TODO(log other failure reason) - tError("failed to create connect: %p", q); + tWarn("failed to create connect:%p", q); taosMemoryFree(buf->base); uv_close((uv_handle_t*)q, NULL); // taosMemoryFree(q); @@ -642,7 +642,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tTrace("conn %p created, fd: %d", pConn, fd); + tTrace("conn %p created, fd:%d", pConn, fd); int addrlen = sizeof(pConn->addr); if (0 != uv_tcp_getpeername(pConn->pTcp, (struct sockaddr*)&pConn->addr, &addrlen)) { @@ -710,7 +710,7 @@ static bool addHandleToAcceptloop(void* arg) { int err = 0; if ((err = uv_tcp_init(srv->loop, &srv->server)) != 0) { - tError("failed to init accept server: %s", uv_err_name(err)); + tError("failed to init accept server:%s", uv_err_name(err)); return false; } @@ -722,11 +722,11 @@ static bool addHandleToAcceptloop(void* arg) { struct sockaddr_in bind_addr; uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) { - tError("failed to bind: %s", uv_err_name(err)); + tError("failed to bind:%s", uv_err_name(err)); return false; } if ((err = uv_listen((uv_stream_t*)&srv->server, 512, uvOnAcceptCb)) != 0) { - tError("failed to listen: %s", uv_err_name(err)); + tError("failed to listen:%s", uv_err_name(err)); terrno = TSDB_CODE_RPC_PORT_EADDRINUSE; return false; } @@ -763,7 +763,7 @@ static SSvrConn* createConn(void* hThrd) { STrans* pTransInst = pThrd->pTransInst; pConn->refId = exh->refId; transRefSrvHandle(pConn); - tTrace("%s handle %p, conn %p created, refId: %" PRId64 "", transLabel(pTransInst), exh, pConn, pConn->refId); + tTrace("%s handle %p, conn %p created, refId:%" PRId64, transLabel(pTransInst), exh, pConn, pConn->refId); return pConn; } @@ -866,10 +866,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, assert(0 == uv_pipe_init(srv->loop, &srv->pipeListen, 0)); #ifdef WINDOWS char pipeName[64]; - snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%p-%lu", taosSafeRand(), GetCurrentProcessId()); + snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%p-" PRIu64, taosSafeRand(), GetCurrentProcessId()); #else char pipeName[PATH_MAX] = {0}; - snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08X-%lu", tsTempDir, TD_DIRSEP, taosSafeRand(), + snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08X-" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(), taosGetSelfPthreadId()); #endif assert(0 == uv_pipe_bind(&srv->pipeListen, pipeName)); @@ -890,17 +890,16 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } int err = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void*)(thrd)); if (err == 0) { - tDebug("sucess to create worker-thread %d", i); - // printf("thread %d create\n", i); + tDebug("success to create worker-thread:%d", i); } else { // TODO: clear all other resource later - tError("failed to create worker-thread %d", i); + tError("failed to create worker-thread:%d", i); goto End; } } if (false == taosValidIpAndPort(srv->ip, srv->port)) { terrno = TAOS_SYSTEM_ERROR(errno); - tError("invalid ip/port, %d:%d, reason: %s", srv->ip, srv->port, terrstr()); + tError("invalid ip/port, %d:%d, reason:%s", srv->ip, srv->port, terrstr()); goto End; } if (false == addHandleToAcceptloop(srv)) { @@ -1021,7 +1020,7 @@ void transRefSrvHandle(void* handle) { return; } int ref = T_REF_INC((SSvrConn*)handle); - tDebug("conn %p ref count: %d", handle, ref); + tDebug("conn %p ref count:%d", handle, ref); } void transUnrefSrvHandle(void* handle) { @@ -1029,7 +1028,7 @@ void transUnrefSrvHandle(void* handle) { return; } int ref = T_REF_DEC((SSvrConn*)handle); - tDebug("conn %p ref count: %d", handle, ref); + tDebug("conn %p ref count:%d", handle, ref); if (ref == 0) { destroyConn((SSvrConn*)handle, true); } diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 8b1dcd46cf..6a4ff213d0 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -153,7 +153,7 @@ int main(int argc, char *argv[]) { dDebugFlag = rpcDebugFlag; uDebugFlag = rpcDebugFlag; } else { - printf("\nusage: %s [options] \n", argv[0]); + printf("\nusage:% [options] \n", argv[0]); printf(" [-p port]: server port number, default is:%d\n", rpcInit.localPort); printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); printf(" [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b5c75ce3c4..18b500aa19 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -94,17 +94,19 @@ static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64 ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr()); + wError("vgId:%d, failed to seek idx file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pRead->pWal->cfg.vgId, ver, + offset, terrstr()); return -1; } SWalIdxEntry entry = {0}; if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) { if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("failed to read idx file, since %s", terrstr()); + wError("vgId:%d, failed to read idx file, since %s", pRead->pWal->cfg.vgId, terrstr()); } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; - wError("read idx file incompletely, read bytes %ld, bytes should be %lu", ret, sizeof(SWalIdxEntry)); + wError("vgId:%d, read idx file incompletely, read bytes %" PRId64 ", bytes should be %" PRIu64, + pRead->pWal->cfg.vgId, ret, sizeof(SWalIdxEntry)); } return -1; } @@ -113,7 +115,8 @@ static int64_t walReadSeekFilePos(SWalReader *pRead, int64_t fileFirstVer, int64 ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("failed to seek log file, ver %ld, pos: %ld, since %s", ver, entry.offset, terrstr()); + wError("vgId:%d, failed to seek log file, index:%" PRId64 ", pos:%" PRId64 ", since %s", pRead->pWal->cfg.vgId, ver, + entry.offset, terrstr()); return -1; } return ret; @@ -129,7 +132,7 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("cannot open file %s, since %s", fnameStr, terrstr()); + wError("vgId:%d, cannot open file %s, since %s", pRead->pWal->cfg.vgId, fnameStr, terrstr()); return -1; } @@ -139,7 +142,7 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("cannot open file %s, since %s", fnameStr, terrstr()); + wError("vgId:%d, cannot open file %s, since %s", pRead->pWal->cfg.vgId, fnameStr, terrstr()); return -1; } @@ -147,13 +150,15 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { return 0; } -static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { +int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { SWal *pWal = pRead->pWal; if (ver == pRead->curVersion) { + wDebug("wal version %ld match, no need to reset", ver); return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { - wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pWal->vers.firstVer, pWal->vers.lastVer); + wError("vgId:$d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId, ver, + pWal->vers.firstVer, pWal->vers.lastVer); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -177,6 +182,8 @@ static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { return -1; } + wDebug("wal version reset from %ld to %ld", pRead->curVersion, ver); + pRead->curVersion = ver; return 0; @@ -187,7 +194,10 @@ void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { int64_t contLen; if (pRead->curVersion != fetchVer) { - if (walReadSeekVer(pRead, fetchVer) < 0) return -1; + if (walReadSeekVer(pRead, fetchVer) < 0) { + ASSERT(0); + return -1; + } } contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead)); if (contLen != sizeof(SWalCkHead)) { @@ -196,6 +206,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + ASSERT(0); pRead->curVersion = -1; return -1; } @@ -220,11 +231,11 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since %s", - pRead->pHead->head.version, ver, tstrerror(terrno)); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", + pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); } else { - wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 ", since file corrupted", - pRead->pHead->head.version, ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", + pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } pRead->curVersion = -1; @@ -233,7 +244,8 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { } if (pReadHead->version != ver) { - wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, + pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; ASSERT(0); @@ -241,7 +253,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { } if (walValidBodyCksum(pRead->pHead) != 0) { - wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; ASSERT(0); @@ -249,6 +261,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { } pRead->curVersion = ver + 1; + wDebug("version advance to %ld, fetch body", pRead->curVersion); return 0; } @@ -261,10 +274,12 @@ static int32_t walSkipFetchBodyNew(SWalReader *pRead) { if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); pRead->curVersion = -1; + ASSERT(0); return -1; } pRead->curVersion++; + wDebug("version advance to %ld, skip fetch", pRead->curVersion); return 0; } @@ -292,7 +307,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { code = walValidHeadCksum(pHead); if (code != 0) { - wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, + ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -338,14 +354,15 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { } if (pReadHead->version != ver) { - wError("wal fetch body error: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, + pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (walValidBodyCksum(*ppHead) != 0) { - wError("wal fetch body error: % " PRId64 ", since body checksum not passed", ver); + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -355,22 +372,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { return 0; } -int32_t walReadWithHandle_s(SWalReader *pRead, int64_t ver, SWalCont **ppHead) { - taosThreadMutexLock(&pRead->mutex); - if (walReadVer(pRead, ver) < 0) { - taosThreadMutexUnlock(&pRead->mutex); - return -1; - } - *ppHead = taosMemoryMalloc(sizeof(SWalCont) + pRead->pHead->head.bodyLen); - if (*ppHead == NULL) { - taosThreadMutexUnlock(&pRead->mutex); - return -1; - } - memcpy(*ppHead, &pRead->pHead->head, sizeof(SWalCont) + pRead->pHead->head.bodyLen); - taosThreadMutexUnlock(&pRead->mutex); - return 0; -} - int32_t walReadVer(SWalReader *pRead, int64_t ver) { int64_t code; @@ -382,14 +383,14 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { // TODO: check wal life if (pRead->curVersion != ver) { if (walReadSeekVer(pRead, ver) < 0) { - wError("unexpected wal log version: % " PRId64 ", since %s", ver, terrstr()); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since %s", pRead->pWal->cfg.vgId, ver, terrstr()); return -1; } } if (ver > pRead->pWal->vers.lastVer || ver < pRead->pWal->vers.firstVer) { - wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pRead->pWal->vers.firstVer, - pRead->pWal->vers.lastVer); + wError("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pRead->pWal->cfg.vgId, ver, + pRead->pWal->vers.firstVer, pRead->pWal->vers.lastVer); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -409,7 +410,8 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { code = walValidHeadCksum(pRead->pHead); if (code != 0) { - wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed", pRead->pWal->cfg.vgId, + ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } @@ -436,8 +438,8 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, - ver); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, + pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -445,7 +447,8 @@ int32_t walReadVer(SWalReader *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); + wError("vgId:%d, unexpected wal log index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, + ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 445cdea45b..900d866a1d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -318,7 +318,8 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); - wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset); + wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, + idxOffset); int64_t size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 7ee73d8e2f..3275774cce 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -210,7 +210,7 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // id = 0; // } // char name[NAME_MAX - 4]; -// snprintf(name, sizeof(name), "/t%ld", id); +// snprintf(name, sizeof(name), "/t" PRId64, id); // p->sem = sem_open(name, O_CREAT | O_EXCL, pshared, value); // p->id = id; // if (p->sem != SEM_FAILED) break; @@ -366,7 +366,7 @@ int32_t tsem_timewait(tsem_t* sem, int64_t nanosecs) { // } // #elif defined(SEM_USE_POSIX) // char name[NAME_MAX - 4]; -// snprintf(name, sizeof(name), "/t%ld", p->id); +// snprintf(name, sizeof(name), "/t" PRId64, p->id); // int r = sem_unlink(name); // if (r) { // int e = errno; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 45a2ffec77..cc823decf4 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -202,9 +202,8 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p, "%lld", pVal); + sscanf(p, "%" PRId64, pVal); #else - // sscanf(p,"%ld",pVal); *pVal = taosStr2Int64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; @@ -237,9 +236,8 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p, "%llu", pVal); + sscanf(p, "%" PRIu64, pVal); #else - // sscanf(p,"%ld",pVal); *pVal = taosStr2UInt64(p, NULL, 10); #endif return TSDB_CODE_SUCCESS; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e009e702d..79f0034052 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -80,7 +80,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim +#./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show diff --git a/tests/script/tsim/sync/vnodesnapshot-restart.sim b/tests/script/tsim/sync/vnodesnapshot-restart.sim new file mode 100644 index 0000000000..3ed82fdfe7 --- /dev/null +++ b/tests/script/tsim/sync/vnodesnapshot-restart.sim @@ -0,0 +1,14 @@ +system sh/stop_dnodes.sh + +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT +#system sh/exec.sh -n dnode2 -s stop -x SIGINT +#system sh/exec.sh -n dnode3 -s stop -x SIGINT +#system sh/exec.sh -n dnode4 -s stop -x SIGINT + diff --git a/tests/script/tsim/sync/vnodesnapshot.sim b/tests/script/tsim/sync/vnodesnapshot.sim index aa42047400..347255b489 100644 --- a/tests/script/tsim/sync/vnodesnapshot.sim +++ b/tests/script/tsim/sync/vnodesnapshot.sim @@ -140,7 +140,6 @@ endi sql create table ct1 using stb tags(1000) -system sh/exec.sh -n dnode4 -s stop -x SIGINT $N = 100 $count = 0 @@ -154,12 +153,8 @@ endw #sql insert into ct1 values(now+0s, 10, 2.0, 3.0) #sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) - #sql flush database db; -#system sh/exec.sh -n dnode4 -s start - -#sql insert into ct1 values(now+1s, 81, 8.1, 8.1)(now+2s, -92, -9.2, -9.2)(now+3s, -73, -7.3, -7.3) @@ -169,7 +164,7 @@ sleep 5000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -#system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT diff --git a/tests/script/tsim/sync/vnodesnapshot2.sim b/tests/script/tsim/sync/vnodesnapshot2.sim new file mode 100644 index 0000000000..c5df5644b1 --- /dev/null +++ b/tests/script/tsim/sync/vnodesnapshot2.sim @@ -0,0 +1,40 @@ +system sh/stop_dnodes.sh + + +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + + + +sleep 5000 + + +sql use db + + + + +$N = 100 +$count = 0 +while $count < $N + $ms = 1591201000000 + $count + sql insert into ct1 values( $ms , $count , 2.1, 3.1) + $count = $count + 1 +endw + + + + + + + +sleep 5000 + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + + + diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 7d8991c095..2c5b8a5d96 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -10,16 +10,19 @@ step1: $x = $x + 1 sleep 1000 if $x == 10 then - print ----> dnode not ready! + print ---> dnode not ready! return -1 endi sql show dnodes -print ----> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data00 $data01 $data02 $data03 $data04 $data05 if $rows != 1 then return -1 endi +if $data(1)[4] != ready then + goto step1 +endi -print =============== step2: create show database +print =============== step2: create db sql create database d1 vgroups 1 buffer 3 sql show databases sql use d1 @@ -32,7 +35,7 @@ if $rows != 1 then return -1 endi -print =============== step4: create show table +print =============== step3: create show table sql create table ct1 using stb tags(1000) sql show tables if $rows != 1 then @@ -40,8 +43,28 @@ if $rows != 1 then endi print =============== step5: insert data +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) print =============== step6: select data +#sql select * from ct1 +#sql select * from stb _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT + +print =============== check +print ----> start to check if there are ERRORS in vagrind log file for each dnode +system_content sh/checkValgrind.sh -n dnode1 + +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +return -1 diff --git a/tests/script/tsim/valgrind/basic2.sim b/tests/script/tsim/valgrind/basic2.sim index 1a76d8ce5c..eb61f85444 100644 --- a/tests/script/tsim/valgrind/basic2.sim +++ b/tests/script/tsim/valgrind/basic2.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect -print =============== step1: show dnodes +print =============== step1: create drop show dnodes $x = 0 step1: $x = $x + 1 @@ -22,75 +22,35 @@ if $data(1)[4] != ready then goto step1 endi -print =============== step2: create alter drop show user -sql create user u1 pass 'taosdata' -sql show users -sql alter user u1 sysinfo 1 -sql alter user u1 enable 1 -sql alter user u1 pass 'taosdata' -sql drop user u1 -sql_error alter user u2 sysinfo 0 - -print =============== step3: create drop dnode -sql create dnode $hostname port 7200 -sql drop dnode 2 -sql alter dnode 1 'debugflag 131' - -print =============== step4: - -print =============== run show xxxx -sql show dnodes -if $rows != 1 then - return -1 -endi - -sql show mnodes -if $rows != 1 then - return -1 -endi - +print =============== step2: create db +sql create database d1 vgroups 1 buffer 3 sql show databases -if $rows != 2 then - return -1 -endi +sql use d1 +sql show vgroups -sql show users +print =============== step3: create show stable +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql show stables if $rows != 1 then return -1 endi -print =============== run select * from information_schema.xxxx -sql select * from information_schema.`dnodes` -if $rows != 1 then - return -1 -endi +print =============== step3: create show table +sql create table ct1 using stb tags(1000) +#sql show tables +#if $rows != 1 then +# return -1 +#endi -sql select * from information_schema.`mnodes` -if $rows != 1 then - return -1 -endi +print =============== step5: insert data +sql insert into ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) -sql select * from information_schema.user_users -if $rows != 1 then - return -1 -endi +print =============== step6: select data +sql select * from ct1 +#sql select * from stb -sql show variables; -if $rows != 4 then - return -1 -endi - -sql show dnode 1 variables; -if $rows <= 0 then - return -1 -endi - -sql show local variables; -if $rows <= 0 then - return -1 -endi - -print =============== stop +_OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check diff --git a/tests/script/tsim/valgrind/basic3.sim b/tests/script/tsim/valgrind/basic3.sim new file mode 100644 index 0000000000..7fba66c468 --- /dev/null +++ b/tests/script/tsim/valgrind/basic3.sim @@ -0,0 +1,152 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode2 -s start -v +sql connect + +print =============== add dnode2 into cluster +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== create database, stable, table +sql create database db vgroups 3 +sql use db +sql create table stb (ts timestamp, c int) tags (t int) +sql create table t0 using stb tags (0) +sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); + +print =============== run show xxxx +sql show dnodes +if $rows != 2 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi + +sql show databases +if $rows != 3 then + return -1 +endi + +sql show stables +if $rows != 1 then + return -1 +endi + +sql show tables +if $rows != 2 then + return -1 +endi + +sql show users +if $rows != 1 then + return -1 +endi + +sql show vgroups +if $rows != 3 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 2 then + return -1 +endi + +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_databases +if $rows != 3 then + return -1 +endi + +sql select * from information_schema.user_stables +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_tables +if $rows != 31 then + return -1 +endi + +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.`vgroups` +if $rows != 3 then + return -1 +endi + +sql show variables; +if $rows != 4 then + return -1 +endi + +sql show dnode 1 variables; +if $rows <= 0 then + return -1 +endi + +sql show local variables; +if $rows <= 0 then + return -1 +endi + +print ==== stop dnode1 and dnode2, and restart dnodes +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +print =============== check dnode1 +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi + +print =============== check dnode2 +system_content sh/checkValgrind.sh -n dnode2 +print cmd return result ----> [ $system_content ] +if $system_content <= 0 then + return 0 +endi + +$null= +if $system_content == $null then + return 0 +endi diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index 62653bbdd3..1a76d8ce5c 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -36,6 +36,60 @@ sql create dnode $hostname port 7200 sql drop dnode 2 sql alter dnode 1 'debugflag 131' +print =============== step4: + +print =============== run show xxxx +sql show dnodes +if $rows != 1 then + return -1 +endi + +sql show mnodes +if $rows != 1 then + return -1 +endi + +sql show databases +if $rows != 2 then + return -1 +endi + +sql show users +if $rows != 1 then + return -1 +endi + +print =============== run select * from information_schema.xxxx +sql select * from information_schema.`dnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.`mnodes` +if $rows != 1 then + return -1 +endi + +sql select * from information_schema.user_users +if $rows != 1 then + return -1 +endi + +sql show variables; +if $rows != 4 then + return -1 +endi + +sql show dnode 1 variables; +if $rows <= 0 then + return -1 +endi + +sql show local variables; +if $rows <= 0 then + return -1 +endi + print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 6bf82b4ef4..5a60dfe254 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -1,13 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v -system sh/exec.sh -n dnode2 -s start -v sql connect -print =============== add dnode2 into cluster -sql create dnode $hostname port 7200 - +print =============== step1: show dnodes $x = 0 step1: $x = $x + 1 @@ -18,16 +15,26 @@ step1: endi sql show dnodes print ---> $data00 $data01 $data02 $data03 $data04 $data05 -print ---> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then +if $rows != 1 then return -1 endi if $data(1)[4] != ready then goto step1 endi -if $data(2)[4] != ready then - goto step1 -endi + +print =============== step2: create alter drop show user +sql create user u1 pass 'taosdata' +sql show users +sql alter user u1 sysinfo 1 +sql alter user u1 enable 1 +sql alter user u1 pass 'taosdata' +sql drop user u1 +sql_error alter user u2 sysinfo 0 + +print =============== step3: create drop dnode +sql create dnode $hostname port 7200 +sql drop dnode 2 +sql alter dnode 1 'debugflag 131' print =============== create database, stable, table sql create database db vgroups 3 @@ -38,7 +45,7 @@ sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); print =============== run show xxxx sql show dnodes -if $rows != 2 then +if $rows != 1 then return -1 endi @@ -67,14 +74,9 @@ if $rows != 1 then return -1 endi -sql show vgroups -if $rows != 3 then - return -1 -endi - print =============== run select * from information_schema.xxxx sql select * from information_schema.`dnodes` -if $rows != 2 then +if $rows != 1 then return -1 endi @@ -123,12 +125,13 @@ if $rows <= 0 then return -1 endi -print ==== stop dnode1 and dnode2, and restart dnodes +print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -print =============== check dnode1 +print =============== check +print ----> start to check if there are ERRORS in vagrind log file for each dnode system_content sh/checkValgrind.sh -n dnode1 + print cmd return result ----> [ $system_content ] if $system_content <= 0 then return 0 @@ -140,17 +143,3 @@ if $system_content == $null then endi return -1 - -print =============== check dnode2 -system_content sh/checkValgrind.sh -n dnode2 -print cmd return result ----> [ $system_content ] -if $system_content <= 0 then - return 0 -endi - -$null= -if $system_content == $null then - return 0 -endi - -return -1 \ No newline at end of file diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 825ac015fb..5735e55c03 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -162,28 +162,28 @@ class TDTestCase: keyDict['h'] = 'abc' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '\'abc\'' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '3' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '\'3\'' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) @@ -193,42 +193,42 @@ class TDTestCase: keyDict['P'] = 'abc' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'abc\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '3' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'3\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '12ab' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'12ab\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) @@ -293,7 +293,7 @@ class TDTestCase: keyDict['p'] = 'errorPassword' retCode, retVal = taos_command(buildPath, "u", keyDict['u'], "taos>", keyDict['c'], sqlString, 'p', keyDict['p']) if retCode == "TAOS_FAIL" and "Authentication failure" in retVal: - tdLog.info("taos -p %s test suceess"%keyDict['p']) + tdLog.info("taos -p %s test success"%keyDict['p']) else: tdLog.exit("taos -u %s -p %s"%(keyDict['u'], keyDict['p'])) diff --git a/tests/system-test/failed.txt b/tests/system-test/failed.txt index d0b66b1769..59c4d625d9 100644 --- a/tests/system-test/failed.txt +++ b/tests/system-test/failed.txt @@ -1 +1,2 @@ #python3 ./test.py -f 2-query/last.py -Q 3 +#./test.sh -f tsim/mnode/basic4.sim diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index da1afe84a7..59908cca8a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,3 +1,27 @@ +IF (TD_WEBSOCKET) + MESSAGE("${Green} use libtaos-ws${ColourReset}") + IF (TD_LINUX) + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo build --release -p taos-ws-sys + COMMAND ./taos-ws-sys/ci/package.sh + INSTALL_COMMAND + COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ENDIF() +ENDIF () + IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/tools/taos_tools/deps/avro/lang/c/src) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/client) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 295fae68b3..2dc5870c4a 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -1,10 +1,19 @@ aux_source_directory(src SHELL_SRC) add_executable(shell ${SHELL_SRC}) + +IF (TD_LINUX AND TD_WEBSOCKET) + ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include -ltaosws) + SET(LINK_WEBSOCKET "-L${CMAKE_BINARY_DIR}/build/lib -ltaosws") + ADD_DEPENDENCIES(shell taosws-rs) +ELSE () + SET(LINK_WEBSOCKET "") +ENDIF () + if(TD_WINDOWS) - target_link_libraries(shell PUBLIC taos_static) + target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() - target_link_libraries(shell PUBLIC taos) + target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET}) endif () target_link_libraries( shell diff --git a/tools/taos-tools b/tools/taos-tools index c9308b0481..9cb71e3c4c 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit c9308b04810faa653548db5f07c753a9d00990eb +Subproject commit 9cb71e3c4c0474553aa961cbe19795541c29b5c7 diff --git a/tools/taosws-rs b/tools/taosws-rs new file mode 160000 index 0000000000..430982a0c2 --- /dev/null +++ b/tools/taosws-rs @@ -0,0 +1 @@ +Subproject commit 430982a0c2c29a819ffc414d11f49f2d424ca3fe