diff --git a/README-CN.md b/README-CN.md index 0b7e42d4fa..bcecc689ed 100644 --- a/README-CN.md +++ b/README-CN.md @@ -41,7 +41,7 @@ TDengine 是一款开源、高性能、云原生的时序数据库 (Time-Series TDengine 目前可以在 Linux、 Windows 等平台上安装和运行。任何 OS 的应用也可以选择 taosAdapter 的 RESTful 接口连接服务端 taosd。CPU 支持 X64/ARM64，后续会支持 MIPS64、Alpha64、ARM32、RISC-V 等 CPU 架构。 -用户可根据需求选择通过源码、[容器](https://docs.taosdata.com/get-started/docker/)、[安装包](https://docs.taosdata.com/get-started/package/)或[Kubenetes](https://docs.taosdata.com/deployment/k8s/)来安装。本快速指南仅适用于通过源码安装。 +用户可根据需求选择通过源码、[容器](https://docs.taosdata.com/get-started/docker/)、[安装包](https://docs.taosdata.com/get-started/package/)或[Kubernetes](https://docs.taosdata.com/deployment/k8s/)来安装。本快速指南仅适用于通过源码安装。 TDengine 还提供一组辅助工具软件 taosTools，目前它包含 taosBenchmark（曾命名为 taosdemo）和 taosdump 两个软件。默认 TDengine 编译不包含 taosTools, 您可以在编译 TDengine 时使用`cmake .. -DBUILD_TOOLS=true` 来同时编译 taosTools。 @@ -104,6 +104,12 @@ sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgco sudo yum config-manager --set-enabled Powertools ``` +### macOS + +``` +sudo brew install argp-standalone pkgconfig +``` + ### 设置 golang 开发环境 TDengine 包含数个使用 Go 语言开发的组件，比如taosAdapter, 请参考 golang.org 官方文档设置 go 开发环境。 @@ -210,14 +216,14 @@ cmake .. -G "NMake Makefiles" nmake ``` - +``` # 安装 @@ -263,6 +269,24 @@ nmake install sudo make install ``` +用户可以在[文件目录结构](https://docs.taosdata.com/reference/directory/)中了解更多在操作系统中生成的目录或文件。 + +从源代码安装也会为 TDengine 配置服务管理 ，用户也可以选择[从安装包中安装](https://docs.taosdata.com/get-started/package/)。 + +安装成功后，可以在应用程序中双击 TDengine 图标启动服务，或者在终端中启动 TDengine 服务： + +```bash +launchctl start taosd +``` + +用户可以使用 TDengine CLI 来连接 TDengine 服务，在终端中，输入： + +```bash +taos +``` + +如果 TDengine CLI 连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印出错误消息。 + ## 快速运行 如果不希望以服务方式运行 TDengine，也可以在终端中直接运行它。也即在生成完成后，执行以下命令（在 Windows 下，生成的可执行文件会带有 .exe 后缀，例如会名为 taosd.exe ）： diff --git a/README.md b/README.md index fe9bb49ed8..6faf386bd6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ English | [简体中文](README-CN.md) | [Learn more about TSDB](https://tdengin # What is TDengine？ -TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages: +TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-series databases with the following advantages: - **[High Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. @@ -105,6 +105,12 @@ If the PowerTools installation fails, you can try to use: sudo yum config-manager --set-enabled powertools ``` +### macOS + +``` +sudo brew install argp-standalone pkgconfig +``` + ### Setup golang environment TDengine includes a few components like taosAdapter developed by Go language. Please refer to golang.org official documentation for golang environment setup. @@ -213,14 +219,14 @@ cmake .. -G "NMake Makefiles" nmake ``` - +``` # Installing @@ -258,7 +264,7 @@ After building successfully, TDengine can be installed by: nmake install ``` - + +Users can find more information about directories installed on the system in the [directory and files](https://docs.tdengine.com/reference/directory/) section. + +Installing from source code will also configure service management for TDengine.Users can also choose to [install from packages](https://docs.tdengine.com/get-started/package/) for it. + +To start the service after installation, double-click the /applications/TDengine to start the program, or in a terminal, use: + +```bash +launchctl start taosd +``` + +Then users can use the TDengine CLI to connect the TDengine server. In a terminal, use: + +```bash +taos +``` + +If TDengine CLI connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. ## Quick Run diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 5d2fcf27b2..2d90d8c49f 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 70f5a1c + GIT_TAG 85179e9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 25a92573fa..4dadbb0151 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -7,7 +7,7 @@ import Tabs from "@theme/Tabs"; import TabItem from "@theme/TabItem"; import PkgListV3 from "/components/PkgListV3"; -This document describes how to install TDengine on Linux and Windows and perform queries and inserts. +This document describes how to install TDengine on Linux/Windows/macOS and perform queries and inserts. - The easiest way to explore TDengine is through [TDengine Cloud](http://cloud.tdengine.com). - To get started with TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). @@ -17,7 +17,7 @@ The full package of TDengine includes the TDengine Server (`taosd`), TDengine Cl The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector. -The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows. +The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on x64 Windows and x64/m1 macOS. ## Installation @@ -111,6 +111,13 @@ Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the W 2. Run the downloaded package to install TDengine. + + + +1. Download the macOS installation package. + +2. Run the downloaded package to install TDengine. + @@ -178,12 +185,33 @@ The following `systemctl` commands can help you manage TDengine service: After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengine Server. + + + + +After the installation is complete, double-click the /applications/TDengine to start the program, or run `launchctl start taosd` to start TDengine Server. + +The following `launchctl` commands can help you manage TDengine service: + +- Start TDengine Server: `launchctl start taosd` + +- Stop TDengine Server: `launchctl stop taosd` + +- Check TDengine Server status: `launchctl list | grep taosd` + +:::info + +- The `launchctl` command does not require _root_ privileges. You don't need to use the `sudo` command. +- The first content returned by the `launchctl list | grep taosd` command is the PID of the program, if '-' indicates that the TDengine service is not running. + +::: + ## Command Line Interface (CLI) -You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line. +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux/macOS terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line. ```bash taos @@ -213,13 +241,13 @@ SELECT * FROM t; Query OK, 2 row(s) in set (0.003128s) ``` -You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either Linux or Windows machines. For more information, see [TDengine CLI](../../reference/taos-shell/). +You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either machines. For more information, see [TDengine CLI](../../reference/taos-shell/). ## Test data insert performance After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance: -Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal. +Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a terminal. ```bash taosBenchmark diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 2c857d37f3..50ee721386 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -10,11 +10,11 @@ import PkgListV3 from "/components/PkgListV3"; 您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. -TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 +TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosdump、TDinsight 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。 -在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。 +在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTools 包获得。TDengine 也提供 Windows x64 平台和 macOS x64/m1 平台的安装包。 ## 安装 @@ -110,6 +110,13 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。 2. 运行可执行程序来安装 TDengine。 + + + +1. 从列表中下载获得 pkg 安装程序； + +2. 运行可执行程序来安装 TDengine。 + @@ -177,12 +184,33 @@ Active: inactive (dead) 安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 + + + + +安装后，在应用程序目录下，双击 TDengine 图标来启动程序，也可以运行 `launchctl start taosd` 来启动 TDengine 服务进程。 + +如下 `launchctl` 命令可以帮助你管理 TDengine 服务： + +- 启动服务进程：`launchctl start taosd` + +- 停止服务进程：`launchctl stop taosd` + +- 查看服务状态：`launchctl list | grep taosd` + +:::info + +- `launchctl` 命令不需要管理员权限，请不要在前面加 `sudo`。 +- `launchctl list | grep taosd` 指令返回的第一个内容是程序的 PID，若为 `-` 则说明 TDengine 服务未运行。 + +::: + ## TDengine 命令行（CLI） -为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux、macOS 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 ```bash taos @@ -212,13 +240,13 @@ SELECT * FROM t; Query OK, 2 row(s) in set (0.003128s) ``` -除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。 +除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。 ## 使用 taosBenchmark 体验写入速度 可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。 -启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）： +启动 TDengine 服务，然后在终端执行 `taosBenchmark`（曾命名为 `taosdemo`）： ```bash $ taosBenchmark @@ -249,7 +277,7 @@ SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters; 查询 location = "California.SanFrancisco" 的记录总条数： ```sql -SELECT COUNT(*) FROM test.meters WHERE location = "Calaifornia.SanFrancisco"; +SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco"; ``` 查询 groupId = 10 的所有记录的平均值、最大值、最小值等： diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index 1828f3317a..95155c3144 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -70,6 +70,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 ## SQL查询示例 - meters 是插入数据的超级表名 - 可以通过超级表的tag来过滤数据，比如查询 `location=California.LosAngeles,groupid=2` 可以通过如下sql： -``` sql -select * from meters where location=California.LosAngeles and groupid=2 + +``` cmd +select * from meters where location="California.LosAngeles" and groupid=2 ``` diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 8d097e3f65..b008953c0b 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -81,6 +81,10 @@ taos> select tbname, * from `meters.current`; t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | Query OK, 4 row(s) in set (0.005399s) ``` -## 查询示例： -想要查询 location=California.LosAngeles groupid=3 的数据，可以通过如下sql: -select * from `meters.voltage` where location="California.LosAngeles" and groupid=3 + +## SQL查询示例 +- `meters.current` 是插入数据的超级表名 +- 可以通过超级表的tag来过滤数据，比如查询 `location=California.LosAngeles groupid=3` 可以通过如下sql： +``` cmd +select * from `meters.current` where location="California.LosAngeles" and groupid=3 +``` diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index e2e7d7c8fa..1795d8edde 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -96,6 +96,9 @@ taos> select * from `meters.current`; Query OK, 2 row(s) in set (0.004076s) ``` -## 查询示例 -想要查询"tags": {"location": "California.LosAngeles", "groupid": 1} 的数据，可以通过如下sql： +## SQL查询示例 +- `meters.voltage` 是插入数据的超级表名 +- 可以通过超级表的tag来过滤数据，比如查询 `location=California.LosAngeles groupid=1` 可以通过如下sql： +``` cmd select * from `meters.voltage` where location="California.LosAngeles" and groupid=1 +``` diff --git a/docs/zh/07-develop/06-stream.md b/docs/zh/07-develop/06-stream.md index c9f1b1d43a..a2e1d1a1da 100644 --- a/docs/zh/07-develop/06-stream.md +++ b/docs/zh/07-develop/06-stream.md @@ -70,7 +70,7 @@ insert into d1004 values("2018-10-03 14:38:06.500", 11.50000, 221, 0.35000); ### 查询以观察结果 ```sql -taos> select start, end, max_current from current_stream_output_stb; +taos> select start, wend, max_current from current_stream_output_stb; start | wend | max_current | =========================================================================== 2018-10-03 14:38:05.000 | 2018-10-03 14:38:10.000 | 10.30000 | diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ac94b0af7d..a1fefa6902 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,6 +27,7 @@ extern "C" { #endif +typedef struct SBuffer SBuffer; typedef struct SSchema SSchema; typedef struct STColumn STColumn; typedef struct STSchema STSchema; @@ -56,6 +57,18 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +// SBuffer ================================ +struct SBuffer { + int64_t nBuf; + uint8_t *pBuf; +}; + +#define tBufferCreate() \ + (SBuffer) { .nBuf = 0, .pBuf = NULL } +void tBufferDestroy(SBuffer *pBuffer); +int32_t tBufferInit(SBuffer *pBuffer, int64_t size); +int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); + // STSchema ================================ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); @@ -162,17 +175,7 @@ struct STSRowBuilder { struct SValue { union { - int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT - uint8_t u8; // TSDB_DATA_TYPE_UTINYINT - int16_t i16; // TSDB_DATA_TYPE_SMALLINT - uint16_t u16; // TSDB_DATA_TYPE_USMALLINT - int32_t i32; // TSDB_DATA_TYPE_INT - uint32_t u32; // TSDB_DATA_TYPE_UINT - int64_t i64; // TSDB_DATA_TYPE_BIGINT - uint64_t u64; // TSDB_DATA_TYPE_UBIGINT - TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP - float f; // TSDB_DATA_TYPE_FLOAT - double d; // TSDB_DATA_TYPE_DOUBLE + int64_t val; struct { uint32_t nData; uint8_t *pData; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index a88f65f6ac..14e9a5af87 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor { char *name; int64_t minValue; int64_t maxValue; - int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); + int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; @@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type); void *getDataMax(int32_t type); - #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 95b1a6ee1d..2bdc99d268 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -38,9 +38,9 @@ extern "C" { #define TD_LOG_DIR_PATH "C:\\TDengine\\log\\" #elif defined(_TD_DARWIN_64) #define TD_TMP_DIR_PATH "/tmp/taosd/" -#define TD_CFG_DIR_PATH "/usr/local/etc/taos/" -#define TD_DATA_DIR_PATH "/usr/local/var/lib/taos/" -#define TD_LOG_DIR_PATH "/usr/local/var/log/taos/" +#define TD_CFG_DIR_PATH "/etc/taos/" +#define TD_DATA_DIR_PATH "/var/lib/taos/" +#define TD_LOG_DIR_PATH "/var/log/taos/" #else #define TD_TMP_DIR_PATH "/tmp/" #define TD_CFG_DIR_PATH "/etc/taos/" diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 5962949a70..38eb0d8fc6 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -23,8 +23,8 @@ extern "C" { #endif #define ENCODE_LIMIT (((uint8_t)1) << 7) -#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode /* ------------------------ LEGACY CODES ------------------------ */ #if 1 @@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { } static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { - *value = ( (((int8_t *)buf)[0] == 0) ? false : true ); + *value = ((((int8_t *)buf)[0] == 0) ? false : true); return POINTER_SHIFT(buf, sizeof(int8_t)); } diff --git a/include/util/tcompression.h b/include/util/tcompression.h index d7ba4dfa3f..84c2922c9e 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -51,287 +51,12 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type); -extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, - const char type); -extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize); -extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, - int32_t outputSize); -extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); -// lossy -extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); -extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); - #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(); void tsCompressExit(); -#endif -static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressBoolImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressBoolImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - return tsCompressStringImp(input, inputSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - return tsDecompressStringImp(input, compressedSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - // lossy mode - if (lossyFloat) { - return tsCompressFloatLossyImp(input, nelements, output); - // lossless mode - } else { -#endif - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - if (lossyDouble) { - // lossy mode - return tsCompressDoubleLossyImp(input, nelements, output); - } else { -#endif - // lossless mode - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -#ifdef TD_TSZ -// -// lossy float double -// static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { @@ -358,33 +83,56 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif -static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressTimestampImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); -static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressTimestampImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} +/************************************************************************* + * STREAM COMPRESSION + *************************************************************************/ +typedef struct SCompressor SCompressor; + +int32_t tCompressorCreate(SCompressor **ppCmprsor); +int32_t tCompressorDestroy(SCompressor *pCmprsor); +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin); +int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus } diff --git a/packaging/tools/TDengine b/packaging/tools/TDengine new file mode 100755 index 0000000000..3025a5639f --- /dev/null +++ b/packaging/tools/TDengine @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + + +function showAlertMessage(){ +osascript < + + + + Label + taosd + ProgramArguments + + /usr/local/bin/taosd + + ProcessType + Interactive + Disabled + + RunAtLoad + + LaunchOnlyOnce + + SessionCreate + + ExitTimeOut + 600 + KeepAlive + + SuccessfulExit + + AfterInitialDemand + + + Program + /usr/local/bin/taosd + + \ No newline at end of file diff --git a/packaging/tools/logo.png b/packaging/tools/logo.png new file mode 100644 index 0000000000..e739e5dd8c Binary files /dev/null and b/packaging/tools/logo.png differ diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt new file mode 100644 index 0000000000..f70ea7161f --- /dev/null +++ b/packaging/tools/mac_before_install.txt @@ -0,0 +1,5 @@ +TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. + +To configure TDengine : edit /etc/taos/taos.cfg +To start service : launchctl start taosd +To access TDengine : use taos in shell \ No newline at end of file diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 58b4385001..a71614ad6e 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -30,36 +30,31 @@ productName="TDengine" emailName="taosdata.com" uninstallScript="rmtaos" +data_dir=${dataDir} +log_dir=${logDir} +cfg_install_dir=${configDir} + if [ "$osType" != "Darwin" ]; then - data_dir=${dataDir} - log_dir=${logDir} - - cfg_install_dir=${configDir} - bin_link_dir="/usr/bin" lib_link_dir="/usr/lib" lib64_link_dir="/usr/lib64" inc_link_dir="/usr/include" - - install_main_dir=${installDir} - - bin_dir="${installDir}/bin" else - data_dir="/usr/local${dataDir}" - log_dir="/usr/local${logDir}" - - cfg_install_dir="/usr/local${configDir}" - bin_link_dir="/usr/local/bin" lib_link_dir="/usr/local/lib" inc_link_dir="/usr/local/include" - install_main_dir="/usr/local/Cellar/tdengine/${verNumber}" - install_main_2_dir="/usr/local/Cellar/tdengine@${verNumber}/${verNumber}" - - bin_dir="/usr/local/Cellar/tdengine/${verNumber}/bin" - bin_2_dir="/usr/local/Cellar/tdengine@${verNumber}/${verNumber}/bin" + if [ -d "/usr/local/Cellar/" ];then + installDir="/usr/local/Cellar/tdengine/${verNumber}" + elif [ -d "/opt/homebrew/Cellar/" ];then + installDir="/opt/homebrew/Cellar/tdengine/${verNumber}" + else + installDir="/usr/local/taos" + fi fi +install_main_dir=${installDir} +bin_dir="${installDir}/bin" +cfg_dir="${installDir}/cfg" service_config_dir="/etc/systemd/system" @@ -71,14 +66,16 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" +csudouser="" +if command -v sudo >/dev/null; then + csudo="sudo " + csudouser="sudo -u ${USER} " +fi service_mod=2 os_type=0 if [ "$osType" != "Darwin" ]; then - if command -v sudo >/dev/null; then - csudo="sudo " - fi initd_mod=0 if pidof systemd &>/dev/null; then service_mod=0 @@ -142,28 +139,16 @@ function kill_taosd() { function install_main_path() { #create install main dir and all sub dir - if [ "$osType" != "Darwin" ]; then - ${csudo}rm -rf ${install_main_dir} || : - ${csudo}mkdir -p ${install_main_dir} - ${csudo}mkdir -p ${install_main_dir}/cfg - ${csudo}mkdir -p ${install_main_dir}/bin - # ${csudo}mkdir -p ${install_main_dir}/connector - ${csudo}mkdir -p ${install_main_dir}/driver - ${csudo}mkdir -p ${install_main_dir}/examples - ${csudo}mkdir -p ${install_main_dir}/include - ${csudo}mkdir -p ${install_main_dir}/share - # ${csudo}mkdir -p ${install_main_dir}/init.d - else - ${csudo}rm -rf ${install_main_dir} || ${csudo}rm -rf ${install_main_2_dir} || : - ${csudo}mkdir -p ${install_main_dir} || ${csudo}mkdir -p ${install_main_2_dir} - ${csudo}mkdir -p ${install_main_dir}/cfg || ${csudo}mkdir -p ${install_main_2_dir}/cfg - ${csudo}mkdir -p ${install_main_dir}/bin || ${csudo}mkdir -p ${install_main_2_dir}/bin - # ${csudo}mkdir -p ${install_main_dir}/connector || ${csudo}mkdir -p ${install_main_2_dir}/connector - ${csudo}mkdir -p ${install_main_dir}/driver || ${csudo}mkdir -p ${install_main_2_dir}/driver - ${csudo}mkdir -p ${install_main_dir}/examples || ${csudo}mkdir -p ${install_main_2_dir}/examples - ${csudo}mkdir -p ${install_main_dir}/include || ${csudo}mkdir -p ${install_main_2_dir}/include - ${csudo}mkdir -p ${install_main_dir}/share || ${csudo}mkdir -p ${install_main_2_dir}/share - fi + ${csudo}rm -rf ${install_main_dir} || : + ${csudo}mkdir -p ${install_main_dir} + ${csudo}mkdir -p ${install_main_dir}/cfg + ${csudo}mkdir -p ${install_main_dir}/bin + # ${csudo}mkdir -p ${install_main_dir}/connector + ${csudo}mkdir -p ${install_main_dir}/driver + ${csudo}mkdir -p ${install_main_dir}/examples + ${csudo}mkdir -p ${install_main_dir}/include + ${csudo}mkdir -p ${install_main_dir}/share + # ${csudo}mkdir -p ${install_main_dir}/init.d } function install_bin() { @@ -175,11 +160,11 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/taosx || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${bin_link_dir}/perfMonitor || : ${csudo}rm -f ${bin_link_dir}/set_core || : - ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : @@ -209,18 +194,26 @@ function install_bin() { [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : else - ${csudo}cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin || ${csudo}cp -r ${binary_dir}/build/bin/* ${install_main_2_dir}/bin || : - ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_2_dir} || : - ${csudo}cp -r ${script_dir}/remove_client.sh ${install_main_dir}/bin || ${csudo}cp -r ${script_dir}/remove_client.sh ${install_main_2_dir}/bin || : - ${csudo}chmod 0555 ${install_main_dir}/bin/* || ${csudo}chmod 0555 ${install_main_2_dir}/bin/* + ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/taosBenchmark ] && ${csudo}cp -r ${binary_dir}/build/bin/taosBenchmark ${install_main_dir}/bin || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || : + ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : + + ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : + ${csudo}chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/${clientName} ] || [ -x ${install_main_2_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || ${csudo}ln -s ${install_main_2_dir}/bin/${clientName} || : - [ -x ${install_main_dir}/bin/${serverName} ] || [ -x ${install_main_2_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || ${csudo}ln -s ${install_main_2_dir}/bin/${serverName} || : - [ -x ${install_main_dir}/bin/taosadapter ] || [ -x ${install_main_2_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || ${csudo}ln -s ${install_main_2_dir}/bin/taosadapter || : - [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : - [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosx ] || [ -x ${install_main_2_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || ln -s ${install_main_2_dir}/bin/taosx ${bin_link_dir}/taosx || : + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || : + [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || : + [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter || : + [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : + [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : fi } @@ -331,28 +324,20 @@ function install_lib() { fi else ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ - ${install_main_dir}/driver || - ${csudo}cp -Rf ${binary_dir}/build/lib/libtaos.${verNumber}.dylib \ - ${install_main_2_dir}/driver && - ${csudo}chmod 777 ${install_main_dir}/driver/* || - ${csudo}chmod 777 ${install_main_2_dir}/driver/* - - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.* \ - ${install_main_dir}/driver/libtaos.1.dylib || - ${csudo}ln -sf ${install_main_2_dir}/driver/libtaos.* \ - ${install_main_2_dir}/driver/libtaos.1.dylib || : - - ${csudo}ln -sf ${install_main_dir}/driver/libtaos.1.dylib \ - ${install_main_dir}/driver/libtaos.dylib || - ${csudo}ln -sf ${install_main_2_dir}/driver/libtaos.1.dylib \ - ${install_main_2_dir}/driver/libtaos.dylib || : + ${install_main_dir}/driver && ${csudo}chmod 777 ${install_main_dir}/driver/* ${csudo}ln -sf ${install_main_dir}/driver/libtaos.${verNumber}.dylib \ - ${lib_link_dir}/libtaos.1.dylib || - ${csudo}ln -sf ${install_main_2_dir}/driver/libtaos.${verNumber}.dylib \ - ${lib_link_dir}/libtaos.1.dylib || : + ${lib_link_dir}/libtaos.1.dylib || : ${csudo}ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib || : + + if [ -f ${binary_dir}/build/lib/libtaosws.dylib ]; then + ${csudo}cp ${binary_dir}/build/lib/libtaosws.dylib \ + ${install_main_dir}/driver && + ${csudo}chmod 777 ${install_main_dir}/driver/libtaosws.dylib ||: + + ${csudo}ln -sf ${install_main_dir}/driver/libtaosws.dylib ${lib_link_dir}/libtaosws.dylib || : + fi fi install_jemalloc @@ -365,37 +350,30 @@ function install_lib() { } function install_header() { + ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : + [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: + ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ + ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* - if [ "$osType" != "Darwin" ]; then - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : - [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ - ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* - - if [ -f ${binary_dir}/build/include/taosws.h ]; then - ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||: - ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||: - fi - - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h - - else - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ - ${install_main_dir}/include || - ${csudo}cp -f ${source_dir}/include/client/taos.h ${source_dir}/include/common/taosdef.h ${source_dir}/include/util/taoserror.h ${source_dir}/include/libs/function/taosudf.h \ - ${install_main_2_dir}/include && - ${csudo}chmod 644 ${install_main_dir}/include/* || ${csudo}chmod 644 ${install_main_2_dir}/include/* + if [ -f ${binary_dir}/build/include/taosws.h ]; then + ${csudo}cp -f ${binary_dir}/build/include/taosws.h ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/taosws.h ||: + ${csudo}ln -sf ${install_main_dir}/include/taosws.h ${inc_link_dir}/taosws.h ||: fi + + ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h + ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h + ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h + + ${csudo}chmod 644 ${install_main_dir}/include/* } function install_config() { if [ ! -f ${cfg_install_dir}/${configFile} ]; then ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/../cfg/${configFile} ] && - ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} + ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_install_dir} && + ${csudo}cp ${script_dir}/../cfg/${configFile} ${cfg_dir} ${csudo}chmod 644 ${cfg_install_dir}/${configFile} ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} @@ -404,6 +382,7 @@ function install_config() { else ${csudo}cp -f ${script_dir}/../cfg/${configFile} \ ${cfg_install_dir}/${configFile}.${verNumber} + ${csudo}cp -f ${script_dir}/../cfg/${configFile} ${cfg_dir} fi } @@ -411,7 +390,8 @@ function install_taosadapter_config() { if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then ${csudo}mkdir -p ${cfg_install_dir} || : [ -f ${binary_dir}/test/cfg/taosadapter.toml ] && - ${csudo}cp ${binary_dir}/test/cfg/taosadapter.toml ${cfg_install_dir} || : + ${csudo}cp ${binary_dir}/test/cfg/taosadapter.toml ${cfg_install_dir} && + ${csudo}cp ${binary_dir}/test/cfg/taosadapter.toml ${cfg_dir} || : [ -f ${cfg_install_dir}/taosadapter.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/taosadapter.toml || : [ -f ${binary_dir}/test/cfg/taosadapter.toml ] && @@ -424,6 +404,7 @@ function install_taosadapter_config() { if [ -f "${binary_dir}/test/cfg/taosadapter.toml" ]; then ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml \ ${cfg_install_dir}/taosadapter.toml.${verNumber} || : + ${csudo}cp -f ${binary_dir}/test/cfg/taosadapter.toml ${cfg_dir} || : fi fi } @@ -431,20 +412,12 @@ function install_taosadapter_config() { function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - if [ "$osType" != "Darwin" ]; then - ${csudo}ln -s ${log_dir} ${install_main_dir}/log - else - ${csudo}ln -s ${log_dir} ${install_main_dir}/log || ${csudo}ln -s ${log_dir} ${install_main_2_dir}/log - fi + ${csudo}ln -s ${log_dir} ${install_main_dir}/log } function install_data() { - ${csudo}mkdir -p ${data_dir} - if [ "$osType" != "Darwin" ]; then - ${csudo}ln -s ${data_dir} ${install_main_dir}/data - else - ${csudo}ln -s ${data_dir} ${install_main_dir}/data || ${csudo}ln -s ${data_dir} ${install_main_2_dir}/data - fi + ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir} + ${csudo}ln -s ${data_dir} ${install_main_dir}/data } function install_connector() { @@ -453,31 +426,17 @@ function install_connector() { else echo "WARNING: go connector not found, please check if want to use it!" fi - if [ "$osType" != "Darwin" ]; then - ${csudo}cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || : - ${csudo}cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &>/dev/null && ${csudo}chmod 777 ${install_main_dir}/connector/*.jar || echo &>/dev/null || : - else - ${csudo}cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || ${csudo}cp -rf ${source_dir}/src/connector/python ${install_main_2_dir}/connector || : - ${csudo}cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &>/dev/null && ${csudo}chmod 777 ${install_main_dir}/connector/*.jar || echo &>/dev/null || : - ${csudo}cp ${binary_dir}/build/lib/*.jar ${install_main_2_dir}/connector &>/dev/null && ${csudo}chmod 777 ${install_main_2_dir}/connector/*.jar || echo &>/dev/null || : - fi + ${csudo}cp -rf ${source_dir}/src/connector/python ${install_main_dir}/connector || : + ${csudo}cp ${binary_dir}/build/lib/*.jar ${install_main_dir}/connector &>/dev/null && ${csudo}chmod 777 ${install_main_dir}/connector/*.jar || echo &>/dev/null || : } function install_examples() { - if [ "$osType" != "Darwin" ]; then - ${csudo}cp -rf ${source_dir}/examples/* ${install_main_dir}/examples || : - else - ${csudo}cp -rf ${source_dir}/examples/* ${install_main_dir}/examples || ${csudo}cp -rf ${source_dir}/examples/* ${install_main_2_dir}/examples || : - fi + ${csudo}cp -rf ${source_dir}/examples/* ${install_main_dir}/examples || : } function install_web() { if [ -d "${binary_dir}/build/share" ]; then - if [ "$osType" != "Darwin" ]; then - ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || : - else - ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_2_dir}/share || : - fi + ${csudo}cp -rf ${binary_dir}/build/share/* ${install_main_dir}/share || : fi } @@ -575,13 +534,36 @@ function install_taosadapter_service() { fi } +function install_service_on_launchctl() { + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : +} + function install_service() { - if ((${service_mod} == 0)); then - install_service_on_systemd - elif ((${service_mod} == 1)); then - install_service_on_sysvinit + if [ "$osType" != "Darwin" ]; then + if ((${service_mod} == 0)); then + install_service_on_systemd + elif ((${service_mod} == 1)); then + install_service_on_sysvinit + else + kill_taosd + fi else - kill_taosd + install_service_on_launchctl + fi +} +function install_app() { + if [ "$osType" = "Darwin" ]; then + ${csudo}rm -rf /Applications/TDengine.app && + ${csudo}mkdir -p /Applications/TDengine.app/Contents/MacOS/ && + ${csudo}cp ${script_dir}/TDengine /Applications/TDengine.app/Contents/MacOS/ && + echo "" | ${csudo}tee /Applications/TDengine.app/Contents/Info.plist > /dev/null && + ${csudo}sips -i ${script_dir}/logo.png > /dev/null && + DeRez -only icns ${script_dir}/logo.png | ${csudo}tee /Applications/TDengine.app/mac_logo.rsrc > /dev/null && + ${csudo}rez -append /Applications/TDengine.app/mac_logo.rsrc -o $'/Applications/TDengine.app/Icon\r' && + ${csudo}SetFile -a C /Applications/TDengine.app/ && + ${csudo}rm /Applications/TDengine.app/mac_logo.rsrc fi } @@ -610,6 +592,7 @@ function update_TDengine() { install_examples install_web install_bin + install_app install_service install_taosadapter_service @@ -633,7 +616,11 @@ function update_TDengine() { [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + if [ "$osType" != "Darwin" ]; then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + else + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi @@ -656,6 +643,7 @@ function install_TDengine() { # install_connector install_examples install_bin + install_app install_service install_taosadapter_service @@ -679,7 +667,11 @@ function install_TDengine() { [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" + if [ "$osType" != "Darwin" ]; then + echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + else + echo -e "${GREEN_DARK}To start service ${NC}: launchctl start ${serverName}${NC}" + fi [ -f ${installDir}/bin/taosadapter ] && \ echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi @@ -694,16 +686,10 @@ echo source directory: $1 echo binary directory: $2 if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m" -elif [ "$osType" != "Darwin" ]; then +else if [ -x ${bin_dir}/${clientName} ]; then update_TDengine else install_TDengine fi -else - if [ -x ${bin_dir}/${clientName} ] || [ -x ${bin_2_dir}/${clientName} ]; then - update_TDengine - else - install_TDengine - fi fi diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index fcc8a2a942..29d4ac017c 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -7,27 +7,52 @@ iplist="" serverFqdn="" -# -----------------------Variables definition--------------------- -script_dir=$(dirname $(readlink -f "$0")) +osType=`uname` + # Dynamic directory data_dir="/var/lib/taos" log_dir="/var/log/taos" -data_link_dir="/usr/local/taos/data" -log_link_dir="/usr/local/taos/log" -install_main_dir="/usr/local/taos" +cfg_install_dir="/etc/taos" + +if [ "$osType" != "Darwin" ]; then + script_dir=$(dirname $(readlink -f "$0")) + verNumber="" + lib_file_ext="so" + + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" + + install_main_dir="/usr/local/taos" +else + script_dir=${source_dir}/packaging/tools + verNumber=`ls tdengine/driver | grep -E "libtaos\.[0-9]\.[0-9]" | sed "s/libtaos.//g" | sed "s/.dylib//g" | head -n 1` + lib_file_ext="dylib" + + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + lib64_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" + + if [ -d "/usr/local/Cellar/" ];then + install_main_dir="/usr/local/Cellar/tdengine/${verNumber}" + elif [ -d "/opt/homebrew/Cellar/" ];then + install_main_dir="/opt/homebrew/Cellar/tdengine/${verNumber}" + else + install_main_dir="/usr/local/taos" + fi +fi + +data_link_dir="${install_main_dir}/data" +log_link_dir="${install_main_dir}/log" # static directory -cfg_dir="/usr/local/taos/cfg" -bin_dir="/usr/local/taos/bin" -lib_dir="/usr/local/taos/driver" -init_d_dir="/usr/local/taos/init.d" -inc_dir="/usr/local/taos/include" - -cfg_install_dir="/etc/taos" -bin_link_dir="/usr/bin" -lib_link_dir="/usr/lib" -lib64_link_dir="/usr/lib64" -inc_link_dir="/usr/include" +cfg_dir="${install_main_dir}/cfg" +bin_dir="${install_main_dir}/bin" +lib_dir="${install_main_dir}/driver" +init_d_dir="${install_main_dir}/init.d" +inc_dir="${install_main_dir}/include" service_config_dir="/etc/systemd/system" @@ -40,8 +65,10 @@ GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' csudo="" +csudouser="" if command -v sudo > /dev/null; then csudo="sudo " + csudouser="sudo -u ${USER} " fi initd_mod=0 @@ -63,6 +90,14 @@ elif $(which service &> /dev/null); then else service_mod=2 fi +if [ "$osType" = "Darwin" ]; then + if [ -d "${install_main_dir}" ];then + ${csudo}rm -rf ${install_main_dir} + fi + ${csudo}mkdir -p ${install_main_dir} + ${csudo}rm -rf ${install_main_dir} + ${csudo}cp -rf tdengine ${install_main_dir} +fi function kill_taosadapter() { # ${csudo}pkill -f taosadapter || : @@ -96,22 +131,24 @@ function install_lib() { ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : - [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : - [ -f ${lib64_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.so || : + [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : + [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo}ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - [ -f ${lib_dir}/libtaosws.so ] && ${csudo}ln -sf ${lib_dir}/libtaosws.so ${lib_link_dir}/libtaosws.so ||: + [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib_link_dir}/libtaosws.${lib_file_ext} ||: if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo}ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : - [ -f ${lib_dir}/libtaosws.so ] && ${csudo}ln -sf ${lib_dir}/libtaosws.so ${lib64_link_dir}/libtaosws.so || : + [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} || : fi - ${csudo}ldconfig + if [ "$osType" != "Darwin" ]; then + ${csudo}ldconfig + fi } function install_bin() { @@ -138,6 +175,7 @@ function install_bin() { [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : + [ -x ${bin_dir}/remove.sh ] && ${csudo}ln -s ${bin_dir}/remove.sh ${bin_link_dir}/rmtaos || : } function add_newHostname_to_hosts() { @@ -466,6 +504,14 @@ function install_service_on_systemd() { ${csudo}systemctl enable taosd } +function install_service_on_launchctl() { + if [ -f ${install_main_dir}/service/com.taosdata.taosd.plist ]; then + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudo}cp ${install_main_dir}/service/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist || : + ${csudouser}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosd.plist || : + fi +} + function install_taosadapter_service() { if ((${service_mod}==0)); then [ -f ${script_dir}/../cfg/taosadapter.service ] &&\ @@ -476,6 +522,7 @@ function install_taosadapter_service() { } function install_service() { + if [ "$osType" != "Darwin" ]; then if ((${service_mod}==0)); then install_service_on_systemd elif ((${service_mod}==1)); then @@ -485,6 +532,25 @@ function install_service() { kill_taosadapter kill_taosd fi + else + install_service_on_launchctl + fi +} + +function install_app() { + if [ "$osType" = "Darwin" ]; then + if [ -f ${install_main_dir}/service/TDengine ]; then + ${csudo}rm -rf /Applications/TDengine.app && + ${csudo}mkdir -p /Applications/TDengine.app/Contents/MacOS/ && + ${csudo}cp ${install_main_dir}/service/TDengine /Applications/TDengine.app/Contents/MacOS/ && + echo "" | ${csudo}tee /Applications/TDengine.app/Contents/Info.plist > /dev/null && + ${csudo}sips -i ${install_main_dir}/service/logo.png > /dev/null && + DeRez -only icns ${install_main_dir}/service/logo.png | ${csudo}tee /Applications/TDengine.app/mac_logo.rsrc > /dev/null && + ${csudo}rez -append /Applications/TDengine.app/mac_logo.rsrc -o $'/Applications/TDengine.app/Icon\r' && + ${csudo}SetFile -a C /Applications/TDengine.app/ && + ${csudo}rm /Applications/TDengine.app/mac_logo.rsrc + fi + fi } function install_TDengine() { @@ -492,7 +558,7 @@ function install_TDengine() { #install log and data dir , then ln to /usr/local/taos ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} - ${csudo}mkdir -p ${data_dir} + ${csudo}mkdir -p ${data_dir} && ${csudo}chmod 777 ${data_dir} ${csudo}rm -rf ${log_link_dir} || : ${csudo}rm -rf ${data_link_dir} || : @@ -508,6 +574,7 @@ function install_TDengine() { install_taosadapter_config install_taosadapter_service install_service + install_app # Ask if to start the service #echo diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index a648750904..6172fc9bc8 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -6,12 +6,31 @@ set -e #set -x verMode=edge +osType=`uname` RED='\033[0;31m' GREEN='\033[1;32m' NC='\033[0m' -installDir="/usr/local/taos" +if [ "$osType" != "Darwin" ]; then + installDir="/usr/local/taos" + bin_link_dir="/usr/bin" + lib_link_dir="/usr/lib" + lib64_link_dir="/usr/lib64" + inc_link_dir="/usr/include" +else + if [ -d "/usr/local/Cellar/" ];then + installDir="/usr/local/Cellar/tdengine/${verNumber}" + elif [ -d "/opt/homebrew/Cellar/" ];then + installDir="/opt/homebrew/Cellar/tdengine/${verNumber}" + else + installDir="/usr/local/taos" + fi + bin_link_dir="/usr/local/bin" + lib_link_dir="/usr/local/lib" + lib64_link_dir="/usr/local/lib" + inc_link_dir="/usr/local/include" +fi serverName="taosd" clientName="taos" uninstallScript="rmtaos" @@ -22,11 +41,8 @@ install_main_dir=${installDir} data_link_dir=${installDir}/data log_link_dir=${installDir}/log cfg_link_dir=${installDir}/cfg -bin_link_dir="/usr/bin" local_bin_link_dir="/usr/local/bin" -lib_link_dir="/usr/lib" -lib64_link_dir="/usr/lib64" -inc_link_dir="/usr/include" + service_config_dir="/etc/systemd/system" taos_service_name=${serverName} @@ -82,6 +98,7 @@ function clean_bin() { # Remove link ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : + ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : @@ -103,7 +120,7 @@ function clean_lib() { [ -f ${lib_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.so || : ${csudo}rm -f ${lib64_link_dir}/libtaos.* || : - [ -f ${lib64_link_dir}/libtaosws.so ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.so || : + [ -f ${lib64_link_dir}/libtaosws.* ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.* || : #${csudo}rm -rf ${v15_java_app_dir} || : } @@ -195,12 +212,20 @@ function clean_service_on_sysvinit() { fi } +function clean_service_on_launchctl() { + ${csudouser}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : + ${csudo}rm /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : +} + function clean_service() { if ((${service_mod} == 0)); then clean_service_on_systemd elif ((${service_mod} == 1)); then clean_service_on_sysvinit else + if [ "$osType" = "Darwin" ]; then + clean_service_on_launchctl + fi kill_taosadapter kill_taosd kill_tarbitrator @@ -241,6 +266,9 @@ elif echo $osinfo | grep -qwi "centos"; then # echo "this is centos system" ${csudo}rpm -e --noscripts tdengine >/dev/null 2>&1 || : fi +if [ "$osType" = "Darwin" ]; then + ${csudo}rm -rf /Applications/TDengine.app +fi echo -e "${GREEN}${productName} is removed successfully!${NC}" echo diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c0ae99806e..110c3bc8f4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,30 @@ #include "tdatablock.h" #include "tlog.h" +// SBuffer ================================ +void tBufferDestroy(SBuffer *pBuffer) { + tFree(pBuffer->pBuf); + pBuffer->pBuf = NULL; +} + +int32_t tBufferInit(SBuffer *pBuffer, int64_t size) { + pBuffer->nBuf = 0; + return tRealloc(&pBuffer->pBuf, size); +} + +int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { + int32_t code = 0; + + code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); + if (code) return code; + + memcpy(pBuffer->pBuf + pBuffer->nBuf, pData, nData); + pBuffer->nBuf += nData; + + return code; +} + +// ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #pragma pack(push, 1) @@ -33,105 +57,21 @@ typedef struct { // SValue int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); + return tPutBinary(p, pValue->pData, pValue->nData); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tPutI8(p ? p + n : p, pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tPutI16(p ? p + n : p, pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tPutI32(p ? p + n : p, pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tPutI64(p ? p + n : p, pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tPutFloat(p ? p + n : p, pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tPutDouble(p ? p + n : p, pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tPutI64(p ? p + n : p, pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tPutU8(p ? p + n : p, pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tPutU16(p ? p + n : p, pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tPutU32(p ? p + n : p, pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tPutU64(p ? p + n : p, pValue->u64); - break; - default: - ASSERT(0); - } + if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, &pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pValue->u64); - break; - default: - ASSERT(0); - } + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { @@ -1219,288 +1159,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag = HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag = HAS_NULL; - } else { - pColData->flag = HAS_VALUE; - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE - int32_t code = 0; - - if (!COL_VAL_IS_NONE(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - } else { - pColData->flag |= HAS_VALUE; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL - int32_t code = 0; - - if (!COL_VAL_IS_NULL(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - pColData->flag |= HAS_VALUE; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pColData->nVal, 2); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE - int32_t code = 0; - - if (!COL_VAL_IS_VALUE(pColVal)) { - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - } else { - pColData->flag |= HAS_NULL; - } - - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } else { - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pColData->nVal, 1); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } - } + pColData->flag = HAS_VALUE; code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - + if (code) return code; pColData->nVal++; - -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NONE; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NULL; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pColData->nVal, 0); + pColData->flag |= HAS_VALUE; - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); } } + code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_NULL; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_VALUE; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + pColData->flag |= HAS_NONE; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pColData->nVal, 2); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; pColData->nVal++; -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, - SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pColData->nVal, 1); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pColData->nVal, 0); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - } code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; pColData->nVal++; -_exit: return code; } -static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { - tColDataAppendValue0, // 0 - tColDataAppendValue1, // HAS_NONE - tColDataAppendValue2, // HAS_NULL - tColDataAppendValue3, // HAS_NULL|HAS_NONE - tColDataAppendValue4, // HAS_VALUE - tColDataAppendValue5, // HAS_VALUE|HAS_NONE - tColDataAppendValue6, // HAS_VALUE|HAS_NULL - tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { + {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 + {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE + {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL + {tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE + {tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE + {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE + {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL + {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE diff --git a/source/common/src/trow.c b/source/common/src/trow.c index f23a2b386e..e1085ce5d8 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); val = varBuf; } else { - val = (const void *)&pColVal->value.i64; + val = (const void *)&pColVal->value.val; } } else { pColVal = NULL; diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 6d77180610..b05ae602f8 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include #include @@ -476,4 +477,5 @@ TEST(testCase, NoneTest) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } +#endif #endif \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 50d26ce9a5..42e56cd488 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + if (pObj->sql != NULL) { + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->ast != NULL) { + if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->physicalPlan != NULL) { + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } int32_t sz = taosArrayGetSize(pObj->tasks); if (tEncodeI32(pEncoder, sz) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 38c7850e6c..8cd8431c34 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SField *pField1 = taosArrayGet(pCreate->pColumns, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfTags; ++i) { SField *pField1 = taosArrayGet(pCreate->pTags, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } } else { - mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name, - createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); + mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d", + createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; goto _OVER; } @@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->dbId = pDb->uid; pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; @@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; pRsp->tableType = TSDB_SUPER_TABLE; @@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); - char rollup[128 + VARSTR_HEADER_SIZE] = {0}; + char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); for (int32_t i = 0; i < rollupNum; ++i) { char *funcName = taosArrayGet(pStb->pFuncs, i); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c6f46b28fc..ea889e7001 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream SStreamObj streamObj = {0}; memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } @@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SName n; int32_t cols = 0; - char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(streamName)); - varDataSetLen(streamName, strlen(varDataVal(streamName))); + char streamName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); @@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN); - varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); char status[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(status, strlen(varDataVal(status))); + char status2[20] = {0}; + mndShowStreamStatus(status2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&status, false); char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(sourceDB)); - varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); + STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(targetDB)); - varDataSetLen(targetDB, strlen(varDataVal(targetDB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); @@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, NULL, true); } else { char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); - varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); } @@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(trigger, strlen(varDataVal(trigger))); + char trigger2[20] = {0}; + mndShowStreamTrigger(trigger2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fec050b5ea..4d228f742a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -96,7 +96,7 @@ void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); -int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); +int32_t metaGetTableTagsByUids(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); @@ -159,7 +159,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); -void* tsdbCacherowsReaderClose(void *pReader); +void *tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 75b2f74096..bf110f1ae3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -38,39 +38,43 @@ extern "C" { goto LABEL; \ } -typedef struct TSDBROW TSDBROW; -typedef struct TABLEID TABLEID; -typedef struct TSDBKEY TSDBKEY; -typedef struct SDelData SDelData; -typedef struct SDelIdx SDelIdx; -typedef struct STbData STbData; -typedef struct SMemTable SMemTable; -typedef struct STbDataIter STbDataIter; -typedef struct SMapData SMapData; -typedef struct SBlockIdx SBlockIdx; -typedef struct SDataBlk SDataBlk; -typedef struct SSttBlk SSttBlk; -typedef struct SDiskDataHdr SDiskDataHdr; -typedef struct SBlockData SBlockData; -typedef struct SDelFile SDelFile; -typedef struct SHeadFile SHeadFile; -typedef struct SDataFile SDataFile; -typedef struct SSttFile SSttFile; -typedef struct SSmaFile SSmaFile; -typedef struct SDFileSet SDFileSet; -typedef struct SDataFWriter SDataFWriter; -typedef struct SDataFReader SDataFReader; -typedef struct SDelFWriter SDelFWriter; -typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; -typedef struct STsdbFS STsdbFS; -typedef struct SRowMerger SRowMerger; -typedef struct STsdbReadSnap STsdbReadSnap; -typedef struct SBlockInfo SBlockInfo; -typedef struct SSmaInfo SSmaInfo; -typedef struct SBlockCol SBlockCol; -typedef struct SVersionRange SVersionRange; -typedef struct SLDataIter SLDataIter; +typedef struct TSDBROW TSDBROW; +typedef struct TABLEID TABLEID; +typedef struct TSDBKEY TSDBKEY; +typedef struct SDelData SDelData; +typedef struct SDelIdx SDelIdx; +typedef struct STbData STbData; +typedef struct SMemTable SMemTable; +typedef struct STbDataIter STbDataIter; +typedef struct SMapData SMapData; +typedef struct SBlockIdx SBlockIdx; +typedef struct SDataBlk SDataBlk; +typedef struct SSttBlk SSttBlk; +typedef struct SDiskDataHdr SDiskDataHdr; +typedef struct SBlockData SBlockData; +typedef struct SDelFile SDelFile; +typedef struct SHeadFile SHeadFile; +typedef struct SDataFile SDataFile; +typedef struct SSttFile SSttFile; +typedef struct SSmaFile SSmaFile; +typedef struct SDFileSet SDFileSet; +typedef struct SDataFWriter SDataFWriter; +typedef struct SDataFReader SDataFReader; +typedef struct SDelFWriter SDelFWriter; +typedef struct SDelFReader SDelFReader; +typedef struct SRowIter SRowIter; +typedef struct STsdbFS STsdbFS; +typedef struct SRowMerger SRowMerger; +typedef struct STsdbReadSnap STsdbReadSnap; +typedef struct SBlockInfo SBlockInfo; +typedef struct SSmaInfo SSmaInfo; +typedef struct SBlockCol SBlockCol; +typedef struct SVersionRange SVersionRange; +typedef struct SLDataIter SLDataIter; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; +typedef struct SDiskDataBuilder SDiskDataBuilder; +typedef struct SBlkInfo SBlkInfo; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -170,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut int32_t aBufN[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr -int32_t tPutDiskDataHdr(uint8_t *p, void *ph); +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); @@ -267,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader @@ -300,7 +305,7 @@ int32_t tsdbMerge(STsdb *pTsdb); #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) -// tsdbCache +// tsdbCache ============================================================================================== int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); @@ -318,6 +323,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); +// tsdbDiskData ============================================================================================== +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma); +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); + // structs ======================= struct STsdbFS { SDelFile *pDelFile; @@ -438,6 +452,17 @@ struct SSmaInfo { int32_t size; }; +struct SBlkInfo { + int64_t minUid; + int64_t maxUid; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + TSDBKEY minTKey; + TSDBKEY maxTKey; +}; + struct SDataBlk { TSDBKEY minKey; TSDBKEY maxKey; @@ -661,6 +686,38 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +struct SDiskCol { + SBlockCol bCol; + const uint8_t *pBit; + const uint8_t *pOff; + const uint8_t *pVal; + SColumnDataAgg agg; +}; + +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aDiskCol; // SArray +}; + +struct SDiskDataBuilder { + int64_t suid; + int64_t uid; + int32_t nRow; + uint8_t cmprAlg; + uint8_t calcSma; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nBuilder; + SArray *aBuilder; // SArray + uint8_t *aBuf[2]; + SDiskData dd; + SBlkInfo bi; +}; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d499db4958..3c9f964ff6 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -207,17 +207,17 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - SMeta *pMeta = mr.pMeta; SMetaReader *pReader = &mr; // query name.idx - if (tdbTbGet(pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { + if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; metaReaderClear(&mr); return -1; } *uid = *(tb_uid_t *)pReader->pBuf; + metaReaderClear(&mr); return 0; @@ -228,11 +228,11 @@ int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - if (metaGetTableEntryByName(&mr, tbName) == 0) { - *tbType = mr.me.type; - } + code = metaGetTableEntryByName(&mr, tbName); + if (code == 0) *tbType = mr.me.type; + metaReaderClear(&mr); - return 0; + return code; } int metaReadNext(SMetaReader *pReader) { @@ -1134,21 +1134,49 @@ END: return ret; } -int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { +static int32_t metaGetTableTagByUid(SMeta *pMeta, uint64_t suid, uint64_t uid, void **tag, int32_t *len, bool lock) { + int ret = 0; + if (lock) { + metaRLock(pMeta); + } + + SCtbIdxKey ctbIdxKey = {.suid = suid, .uid = uid}; + ret = tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), tag, len); + if (lock) { + metaULock(pMeta); + } + + return ret; +} +int32_t metaGetTableTagsByUids(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { + const int32_t LIMIT = 128; + + int32_t isLock = false; int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; for (int i = 0; i < sz; i++) { - tb_uid_t *id = taosArrayGet(uidList, i); - SCtbIdxKey ctbIdxKey = {.suid = suid, .uid = *id}; + tb_uid_t *id = taosArrayGet(uidList, i); - void *val = NULL; - int32_t len = 0; - if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL && - 0 == tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), &val, &len)) { - taosHashPut(tags, id, sizeof(tb_uid_t), val, len); + if (i % LIMIT == 0) { + if (isLock) metaULock(pMeta); + + metaRLock(pMeta); + isLock = true; + } + + if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL) { + void *val = NULL; + int32_t len = 0; + if (metaGetTableTagByUid(pMeta, suid, *id, &val, &len, false) == 0) { + taosHashPut(tags, id, sizeof(tb_uid_t), val, len); + tdbFree(val); + } } } + if (isLock) metaULock(pMeta); + return 0; } + int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a921349e6..05c941cc4d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs}); + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); } @@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); *ppRow = NULL; return code; @@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; @@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { goto _next_fileset; } - tMapDataReset(&state->blockMap); + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; @@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + return code; } @@ -1052,7 +1059,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo lastRowTs = TSDBROW_TS(pRow); STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1151,7 +1158,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { lastRowTs = rowTs; STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 5403395623..995d7a3155 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -17,6 +17,8 @@ typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; +#define USE_STREAM_COMPRESSION 0 + typedef struct { SRBTreeNode n; SRowInfo r; @@ -75,7 +77,11 @@ typedef struct { SArray *aSttBlk; // SArray SMapData mBlock; // SMapData SBlockData bData; - SBlockData bDatal; +#if USE_STREAM_COMPRESSION + SDiskDataBuilder *pBuilder; +#else + SBlockData bDatal; +#endif } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -115,33 +121,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) { int32_t tsdbBegin(STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; if (!pTsdb) return code; SMemTable *pMemTable; code = tsdbMemTableCreate(pTsdb, &pMemTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock - code = taosThreadRwlockWrlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->mem = pMemTable; // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d, tsdb begin failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -149,6 +154,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { if (!pTsdb) return 0; int32_t code = 0; + int32_t lino = 0; SCommitter commith; SMemTable *pMemTable = pTsdb->mem; @@ -164,76 +170,74 @@ int32_t tsdbCommit(STsdb *pTsdb) { // start commit code = tsdbStartCommit(pTsdb, &commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit impl code = tsdbCommitData(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbCommitDel(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end commit code = tsdbEndCommit(&commith, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbEndCommit(&commith, code); - tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbEndCommit(&commith, code); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdx == NULL) { + if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData)); - if (pCommitter->aDelData == NULL) { + if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdxN == NULL) { + if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } SDelFile *pDelFileR = pCommitter->fs.pDelFile; if (pDelFileR) { code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // prepare new SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, commit del start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); + } return code; } static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { int32_t code = 0; + int32_t lino = 0; SDelData *pDelData; tb_uid_t suid; tb_uid_t uid; @@ -252,7 +256,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel uid = pDelIdx->uid; code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); } @@ -266,62 +270,65 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel for (; pDelData; pDelData = pDelData->pNext) { if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } // write code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put delIdx if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit table del failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->pDelFReader) { code = tsdbDelFReaderClose(&pCommitter->pDelFReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayDestroy(pCommitter->aDelIdx); taosArrayDestroy(pCommitter->aDelData); taosArrayDestroy(pCommitter->aDelIdxN); - return code; - -_err: - tsdbError("vgId:%d, commit del end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t code = 0; + int32_t lino = 0; if (suid) { if (pSkmInfo->suid == suid) { @@ -336,7 +343,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->uid = uid; tTSchemaDestroy(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -344,6 +351,7 @@ _exit: static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->skmRow.pTSchema) { if (pCommitter->skmRow.suid == suid) { @@ -359,9 +367,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.uid = uid; tTSchemaDestroy(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -369,6 +375,7 @@ _exit: static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; ASSERT(pCommitter->dReader.pBlockIdx); @@ -378,7 +385,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); ASSERT(pCommitter->dReader.mBlock.nItem > 0); } else { @@ -391,6 +398,7 @@ _exit: static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; pCommitter->pIter = NULL; tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); @@ -431,14 +439,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iStt = iStt; code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; pIter->iSttBlk = 0; SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -460,16 +468,19 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { } code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; @@ -484,17 +495,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pCommitter->dReader.iBlockIdx = 0; if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { pCommitter->dReader.pBlockIdx = NULL; } @@ -531,28 +542,32 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } wSet.aSttF[wSet.nSttF - 1] = &fStt; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); +#if USE_STREAM_COMPRESSION + tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); +#else tBlockDataReset(&pCommitter->dWriter.bDatal); +#endif // open iter code = tsdbOpenCommitIter(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; if (pBlockData->nRow == 0) return code; @@ -586,24 +601,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa dataBlk.nSubBlock++; code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put SDataBlk code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // clear tBlockDataClear(pBlockData); - return code; - -_err: - tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; SSttBlk sstBlk; if (pBlockData->nRow == 0) return code; @@ -626,114 +642,159 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray // write code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // push SSttBlk if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // clear tBlockDataClear(pBlockData); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; +} -_err: - tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) { + int32_t code = 0; + int32_t lino = 0; + + if (pBuilder->nRow == 0) return code; + + // gnrt + const SDiskData *pDiskData; + const SBlkInfo *pBlkInfo; + code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + SSttBlk sttBlk = {.suid = pBuilder->suid, + .minUid = pBlkInfo->minUid, + .maxUid = pBlkInfo->maxUid, + .minKey = pBlkInfo->minKey, + .maxKey = pBlkInfo->maxKey, + .minVer = pBlkInfo->minVer, + .maxVer = pBlkInfo->maxVer, + .nRow = pBuilder->nRow}; + // write + code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + // push + if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // clear + tDiskDataBuilderClear(pBuilder); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // write aSttBlk code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update file header code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // upsert SDFileSet code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // close and sync code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dReader.pReader) { code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; + int32_t lino = 0; while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; // commit file data start code = tsdbCommitFileDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl code = tsdbCommitFileDataImpl(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit file data end code = tsdbCommitFileDataEnd(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbDataFReaderClose(&pCommitter->dReader.pReader); + tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); + } return code; } // ---------------------------------------------------------------------------- static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; memset(pCommitter, 0, sizeof(*pCommitter)); ASSERT(pTsdb->mem && pTsdb->imem == NULL); @@ -754,30 +815,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dReader.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { @@ -785,33 +847,41 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pIter->bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } // writer pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pCommitter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dWriter.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION + code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); +#else code = tBlockDataCreate(&pCommitter->dWriter.bDatal); - if (code) goto _exit; +#endif + TSDB_CHECK_CODE(code, lino, _exit); _exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } @@ -833,13 +903,19 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); +#if USE_STREAM_COMPRESSION + tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); +#else tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); +#endif tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } static int32_t tsdbCommitData(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -848,30 +924,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { // start ==================== code = tsdbCommitDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl ==================== pCommitter->nextKey = pMemTable->minKey; while (pCommitter->nextKey < TSKEY_MAX) { code = tsdbCommitFileData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // end ==================== tsdbCommitDataEnd(pCommitter); _exit: - tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); - return code; - -_err: - tsdbCommitDataEnd(pCommitter); - tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDel(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -882,7 +957,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // start code = tsdbCommitDelStart(pCommitter); if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // impl @@ -918,7 +993,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_del: code = tsdbCommitTableDel(pCommitter, pTbData, NULL); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -926,7 +1001,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_disk_del: code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iDelIdx++; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; @@ -934,7 +1009,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_and_disk_del: code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -945,28 +1020,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // end code = tsdbCommitDelEnd(pCommitter); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); - return code; - -_err: - tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); + } return code; } static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; ASSERT(eno == 0); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock taosThreadRwlockWrlock(&pTsdb->rwLock); @@ -975,7 +1050,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->imem = NULL; @@ -987,16 +1062,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); - // if (pCommitter->toMerge) { - // code = tsdbMerge(pTsdb); - // if (code) goto _err; - // } - - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); + } return code; } @@ -1008,6 +1079,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->pIter) { SDataIter *pIter = pCommitter->pIter; @@ -1085,11 +1157,17 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { } _exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; @@ -1098,13 +1176,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) while (pRowInfo) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1119,29 +1197,32 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBlockData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData; code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); tBlockDataClear(pBDataW); int32_t iRow = 0; @@ -1152,7 +1233,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); if (c < 0) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1163,13 +1244,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } else if (c > 0) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1186,13 +1267,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } while (pRow) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1203,22 +1284,25 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); @@ -1237,7 +1321,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { if (c < 0) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1247,13 +1331,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } } else if (c > 0) { code = tsdbCommitAheadBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } else { code = tsdbCommitMergeBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1268,7 +1352,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { while (pDataBlk) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1279,70 +1363,113 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } -static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { +static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pBDatal->suid || pBDatal->uid) { - if ((pBDatal->suid != id.suid) || (id.suid == 0)) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _exit; - tBlockDataReset(pBDatal); +#if USE_STREAM_COMPRESSION + SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; + if (pBuilder->suid || pBuilder->uid) { + if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + tDiskDataBuilderClear(pBuilder); } } - if (!pBDatal->suid && !pBDatal->uid) { + if (!pBuilder->suid && !pBuilder->uid) { + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } +#else + SBlockData *pBData = &pCommitter->dWriter.bDatal; + if (pBData->suid || pBData->uid) { + if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataReset(pBData); + } + } + + if (!pBData->suid && !pBData->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; - code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _exit; + code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); } +#endif _exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); - if (code) goto _err; - if (pBDatal->nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; +#if USE_STREAM_COMPRESSION + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); } +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif } - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { @@ -1351,67 +1478,102 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; - SBlockData *pBData; if (pCommitter->toLastOnly) { - pBData = &pCommitter->dWriter.bDatal; - code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + TSDB_CHECK_CODE(code, lino, _exit); + pTSchema = pCommitter->skmRow.pTSchema; + } + +#if USE_STREAM_COMPRESSION + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid); +#endif + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + +#if USE_STREAM_COMPRESSION + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); + } +#else + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif + } } else { - pBData = &pCommitter->dWriter.bData; + SBlockData *pBData = &pCommitter->dWriter.bData; ASSERT(pBData->nRow == 0); - } - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; - pTSchema = pCommitter->skmRow.pTSchema; - } + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + TSDB_CHECK_CODE(code, lino, _exit); + pTSchema = pCommitter->skmRow.pTSchema; + } - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - if (code) goto _err; + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } - if (pBData->nRow >= pCommitter->maxRow) { - if (pCommitter->toLastOnly) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; - } else { + if (pBData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbAppendLastBlock(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); } } } - if (!pCommitter->toLastOnly && pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; - } else { - code = tsdbAppendLastBlock(pCommitter); - if (code) goto _err; - } - } - _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo; TABLEID id = {0}; @@ -1421,36 +1583,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.uid = pRowInfo->uid; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // start tMapDataReset(&pCommitter->dWriter.mBlock); // impl code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* handle remain table data */ code = tsdbCommitTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -1458,15 +1620,20 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.suid = INT64_MAX; id.uid = INT64_MAX; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); +#else code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; +#endif + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 3bd71f0ea6..43be51a694 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -16,69 +16,682 @@ #include "tsdb.h" typedef struct SDiskColBuilder SDiskColBuilder; + struct SDiskColBuilder { - uint8_t flags; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; + int16_t cid; + int8_t type; + uint8_t cmprAlg; + uint8_t calcSma; + int8_t flag; + int32_t nVal; + uint8_t *pBitMap; + int32_t offset; + SCompressor *pOffC; + SCompressor *pValC; + SColumnDataAgg sma; + uint8_t minSet; + uint8_t maxSet; + uint8_t *aBuf[2]; }; -int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { +// SDiskData ================================================ +static int32_t tDiskDataDestroy(SDiskData *pDiskData) { int32_t code = 0; - // TODO + pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); return code; } -// ================================================================ -typedef struct SDiskDataBuilder SDiskDataBuilder; -struct SDiskDataBuilder { - SDiskDataHdr hdr; - SArray *aBlockCol; // SArray -}; +// SDiskColBuilder ================================================ +#define tDiskColBuilderCreate() \ + (SDiskColBuilder) { 0 } -int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { - int32_t code = 0; - // TODO - return code; -} - -void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - // TODO -} - -void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { - pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // - .fmtVer = 0, - .suid = suid, - .uid = uid, - .cmprAlg = cmprAlg}; -} - -void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { - // TODO -} - -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { +static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { int32_t code = 0; - // uid (todo) - - // version (todo) - - // TSKEY (todo) - - SRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); - - for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { + tFree(pBuilder->pBitMap); + if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); + if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); } return code; } -int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { +static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg, + uint8_t calcSma) { int32_t code = 0; - // TODO + + pBuilder->cid = cid; + pBuilder->type = type; + pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma; + pBuilder->flag = 0; + pBuilder->nVal = 0; + pBuilder->offset = 0; + + if (IS_VAR_DATA_TYPE(type)) { + if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; + code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + if (code) return code; + } + + if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; + code = tCompressStart(pBuilder->pValC, type, cmprAlg); + if (code) return code; + + if (pBuilder->calcSma) { + pBuilder->sma = (SColumnDataAgg){.colId = cid}; + pBuilder->minSet = 0; + pBuilder->maxSet = 0; + } + return code; -} \ No newline at end of file +} + +static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { + int32_t code = 0; + + ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); + + *pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid, + .type = pBuilder->type, + .smaOn = pBuilder->calcSma, + .flag = pBuilder->flag, + .szOrigin = 0, + .szBitmap = 0, + .szOffset = 0, + .szValue = 0, + .offset = 0}, + .pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma}; + + if (pBuilder->flag == HAS_NULL) return code; + + // BITMAP + if (pBuilder->flag != HAS_VALUE) { + int32_t nBit; + if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + nBit = BIT2_SIZE(pBuilder->nVal); + } else { + nBit = BIT1_SIZE(pBuilder->nVal); + } + + code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + pDiskCol->bCol.szBitmap = + tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES, + pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); + pDiskCol->pBit = pBuilder->aBuf[0]; + } + + // OFFSET + if (IS_VAR_DATA_TYPE(pBuilder->type)) { + code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL); + if (code) return code; + } + + // VALUE + if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { + code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin); + if (code) return code; + } + + return code; +} + +static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColVal->type)) { + code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); + if (code) return code; + pBuilder->offset += pColVal->value.nData; + + code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); + if (code) return code; + } else { + code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); + if (code) return code; + } + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_VALUE; + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NONE; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NULL; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + // bit map + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + // value + pBuilder->flag |= HAS_VALUE; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; + } + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + pBuilder->flag |= HAS_NULL; + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + pBuilder->flag |= HAS_VALUE; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; + } + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + pBuilder->flag |= HAS_NONE; + + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pBuilder->nVal, 2); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; + } + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) { + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NONE; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pBuilder->nVal, 1); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pBuilder->nVal, 0); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { + {tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0 + {tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE + {tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL + {tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE + {tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE + {tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE + {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL + {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE +}; +extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); +static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + if (pBuilder->calcSma) { + if (COL_VAL_IS_VALUE(pColVal)) { + tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); + } else { + pBuilder->sma.numOfNull++; + } + } + + if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) { + code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal); + if (code) return code; + } + + pBuilder->nVal++; + + return code; +} + +// SDiskDataBuilder ================================================ +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { + int32_t code = 0; + + *ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder)); + if (*ppBuilder == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + return code; +} + +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { + if (pBuilder == NULL) return NULL; + + if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); + if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); + if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); + + if (pBuilder->aBuilder) { + for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); + tDiskColBuilderDestroy(pDCBuilder); + } + taosArrayDestroy(pBuilder->aBuilder); + } + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); + } + tDiskDataDestroy(&pBuilder->dd); + taosMemoryFree(pBuilder); + + return NULL; +} + +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma) { + int32_t code = 0; + + ASSERT(pId->suid || pId->uid); + + pBuilder->suid = pId->suid; + pBuilder->uid = pId->uid; + pBuilder->nRow = 0; + pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = calcSma; + pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, + .maxUid = INT64_MIN, + .minKey = TSKEY_MAX, + .maxKey = TSKEY_MIN, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN, + .minTKey = TSDBKEY_MAX, + .maxTKey = TSDBKEY_MIN}; + + if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; + code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; + code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; + code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + if (code) return code; + + if (pBuilder->aBuilder == NULL) { + pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); + if (pBuilder->aBuilder == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + pBuilder->nBuilder = 0; + for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + STColumn *pTColumn = &pTSchema->columns[iCol]; + + if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { + SDiskColBuilder dc = tDiskColBuilderCreate(); + if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); + + code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg, + (calcSma && (pTColumn->flags & COL_SMA_ON))); + if (code) return code; + + pBuilder->nBuilder++; + } + + return code; +} + +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { + int32_t code = 0; + pBuilder->suid = 0; + pBuilder->uid = 0; + pBuilder->nRow = 0; + return code; +} + +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { + int32_t code = 0; + + ASSERT(pBuilder->suid || pBuilder->uid); + ASSERT(pId->suid == pBuilder->suid); + + TSDBKEY kRow = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow; + if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow; + + // uid + if (pBuilder->uid && pBuilder->uid != pId->uid) { + ASSERT(pBuilder->suid); + for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { + code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); + if (code) return code; + } + pBuilder->uid = 0; + } + if (pBuilder->uid == 0) { + code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); + if (code) return code; + } + if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid; + if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; + + // version + code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t)); + if (code) return code; + if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version; + if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version; + + // TSKEY + code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t)); + if (code) return code; + if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; + if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; + + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); + + SColVal *pColVal = tRowIterNext(&iter); + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); + + while (pColVal && pColVal->cid < pDCBuilder->cid) { + pColVal = tRowIterNext(&iter); + } + + if (pColVal && pColVal->cid == pDCBuilder->cid) { + code = tDiskColAddVal(pDCBuilder, pColVal); + if (code) return code; + pColVal = tRowIterNext(&iter); + } else { + code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); + if (code) return code; + } + } + pBuilder->nRow++; + + return code; +} + +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) { + int32_t code = 0; + + ASSERT(pBuilder->nRow); + + *ppDiskData = NULL; + *ppBlkInfo = NULL; + + SDiskData *pDiskData = &pBuilder->dd; + // reset SDiskData + pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBuilder->suid, + .uid = pBuilder->uid, + .szUid = 0, + .szVer = 0, + .szKey = 0, + .szBlkCol = 0, + .nRow = pBuilder->nRow, + .cmprAlg = pBuilder->cmprAlg}; + pDiskData->pUid = NULL; + pDiskData->pVer = NULL; + pDiskData->pKey = NULL; + + // UID + if (pBuilder->uid == 0) { + code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL); + if (code) return code; + } + + // VERSION + code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL); + if (code) return code; + + // TSKEY + code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL); + if (code) return code; + + // aDiskCol + if (pDiskData->aDiskCol) { + taosArrayClear(pDiskData->aDiskCol); + } else { + pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol)); + if (pDiskData->aDiskCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + int32_t offset = 0; + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); + + if (pDCBuilder->flag == HAS_NONE) continue; + + SDiskCol dCol; + + code = tGnrtDiskCol(pDCBuilder, &dCol); + if (code) return code; + + dCol.bCol.offset = offset; + offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue; + + if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); + } + + *ppDiskData = pDiskData; + *ppBlkInfo = &pBuilder->bi; + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c3cb5f9eb8..de64b8a61e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } } + // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, i); colDataAppendNNULL(pColData, 0, remain); @@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; + // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { -// int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; -// setBlockAllDumped(pDumpInfo, ts, pReader->order); + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (outOfTimeWindow(ts, &pReader->window)) { // the remain data has out of query time window, ignore current block + setBlockAllDumped(pDumpInfo, ts, pReader->order); + } } else { - int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; - setBlockAllDumped(pDumpInfo, k, pReader->order); + int64_t ts = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; + setBlockAllDumped(pDumpInfo, ts, pReader->order); } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index fc577e3962..167b4a104c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -128,7 +128,7 @@ _exit: return code; } -static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); @@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, // write if (pSmaInfo->size) { - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; @@ -607,6 +604,132 @@ _err: return code; } +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { + int32_t code = 0; + int32_t lino = 0; + + STsdbFD *pFD = NULL; + if (pSmaInfo) { + pFD = pWriter->pDataFD; + pBlkInfo->offset = pWriter->fData.size; + } else { + pFD = pWriter->pSttFD; + pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; + } + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; + + // hdr + int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); + code = tRealloc(&pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + + tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); + + code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + pBlkInfo->szKey += n; + pBlkInfo->szBlock += n; + + // uid + ver + key + if (pDiskData->pUid) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); + TSDB_CHECK_CODE(code, lino, _exit); + pBlkInfo->szKey += pDiskData->hdr.szUid; + pBlkInfo->szBlock += pDiskData->hdr.szUid; + } + + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); + TSDB_CHECK_CODE(code, lino, _exit); + pBlkInfo->szKey += pDiskData->hdr.szVer; + pBlkInfo->szBlock += pDiskData->hdr.szVer; + + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + pBlkInfo->szKey += pDiskData->hdr.szKey; + pBlkInfo->szBlock += pDiskData->hdr.szKey; + + // aBlockCol + if (pDiskData->hdr.szBlkCol) { + code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + n = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); + } + ASSERT(n == pDiskData->hdr.szBlkCol); + + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; + } + + // aDiskCol + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (pDiskCol->pBit) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; + } + + if (pDiskCol->pOff) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlkInfo->szBlock += pDiskCol->bCol.szOffset; + } + + if (pDiskCol->pVal) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); + TSDB_CHECK_CODE(code, lino, _exit); + + pBlkInfo->szBlock += pDiskCol->bCol.szValue; + } + } + + if (pSmaInfo) { + pWriter->fData.size += pBlkInfo->szBlock; + } else { + pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; + goto _exit; + } + + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; + if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; + if (!pDiskCol->bCol.smaOn) continue; + + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); + TSDB_CHECK_CODE(code, lino, _exit); + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); + } + + if (pSmaInfo->size) { + pSmaInfo->offset = pWriter->fSma.size; + + code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); + TSDB_CHECK_CODE(code, lino, _exit); + + pWriter->fSma.size += pSmaInfo->size; + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; @@ -946,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); - ASSERT(pBlockData->uid == hdr.uid); + pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; // uid diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4999e7a49a..47c836d9c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { STColumn *pTColumn; int32_t iCol, jCol = 1; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { pTColumn = &pMerger->pTSchema->columns[iCol]; @@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); @@ -1105,9 +1105,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pColVal = tRowIterNext(&rIter); } } - -_exit: pBlockData->nRow++; + return code; _err: @@ -1455,9 +1454,8 @@ _exit: } // SDiskDataHdr ============================== -int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { - int32_t n = 0; - SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) { + int32_t n = 0; n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32v(p ? p + n : p, pHdr->fmtVer); @@ -1516,174 +1514,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { return n; } +#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \ + do { \ + (SUM_V) += (VAL); \ + if (!(MINSET)) { \ + (MIN_V) = (VAL); \ + (MINSET) = 1; \ + } else if ((MIN_V) > (VAL)) { \ + (MIN_V) = (VAL); \ + } \ + if (!(MAXSET)) { \ + (MAX_V) = (VAL); \ + (MAXSET) = 1; \ + } else if ((MAX_V) < (VAL)) { \ + (MAX_V) = (VAL); \ + } \ + } while (0) + +static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int16_t val = *(int16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int32_t val = *(int32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int64_t val = *(int64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + float val = *(float *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + double val = *(double *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint8_t val = *(uint8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint16_t val = *(uint16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + uint32_t val = *(uint32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint64_t val = *(uint64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = { + NULL, + tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL + tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT + tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT + tSmaUpdateInt, // TSDB_DATA_TYPE_INT + tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT + tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT + tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT + tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT + tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT + tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB +}; void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { - SColVal colVal; - SColVal *pColVal = &colVal; - - memset(pColAgg, 0, sizeof(*pColAgg)); - bool minAssigned = false; - bool maxAssigned = false; - *pColAgg = (SColumnDataAgg){.colId = pColData->cid}; - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - tColDataGetValue(pColData, iVal, pColVal); + uint8_t minSet = 0; + uint8_t maxSet = 0; - if (!COL_VAL_IS_VALUE(pColVal)) { - pColAgg->numOfNull++; + SColVal cv; + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + tColDataGetValue(pColData, iVal, &cv); + + if (COL_VAL_IS_VALUE(&cv)) { + tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); } else { - switch (pColData->type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - break; - case TSDB_DATA_TYPE_TINYINT: { - pColAgg->sum += colVal.value.i8; - if (!minAssigned || pColAgg->min > colVal.value.i8) { - pColAgg->min = colVal.value.i8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.i8) { - pColAgg->max = colVal.value.i8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - pColAgg->sum += colVal.value.i16; - if (!minAssigned || pColAgg->min > colVal.value.i16) { - pColAgg->min = colVal.value.i16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.i16) { - pColAgg->max = colVal.value.i16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_INT: { - pColAgg->sum += colVal.value.i32; - if (!minAssigned || pColAgg->min > colVal.value.i32) { - pColAgg->min = colVal.value.i32; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.i32) { - pColAgg->max = colVal.value.i32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - pColAgg->sum += colVal.value.i64; - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - *(double *)(&pColAgg->sum) += colVal.value.f; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) { - *(double *)(&pColAgg->min) = colVal.value.f; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) { - *(double *)(&pColAgg->max) = colVal.value.f; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)(&pColAgg->sum) += colVal.value.d; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) { - *(double *)(&pColAgg->min) = colVal.value.d; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) { - *(double *)(&pColAgg->max) = colVal.value.d; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_VARCHAR: - break; - case TSDB_DATA_TYPE_TIMESTAMP: { - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_NCHAR: - break; - case TSDB_DATA_TYPE_UTINYINT: { - pColAgg->sum += colVal.value.u8; - if (!minAssigned || pColAgg->min > colVal.value.u8) { - pColAgg->min = colVal.value.u8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.u8) { - pColAgg->max = colVal.value.u8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - pColAgg->sum += colVal.value.u16; - if (!minAssigned || pColAgg->min > colVal.value.u16) { - pColAgg->min = colVal.value.u16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.u16) { - pColAgg->max = colVal.value.u16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - pColAgg->sum += colVal.value.u32; - if (!minAssigned || pColAgg->min > colVal.value.u32) { - pColAgg->min = colVal.value.u32; - minAssigned = true; - } - if (!minAssigned || pColAgg->max < colVal.value.u32) { - pColAgg->max = colVal.value.u32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - pColAgg->sum += colVal.value.u64; - if (!minAssigned || pColAgg->min > colVal.value.u64) { - pColAgg->min = colVal.value.u64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < colVal.value.u64) { - pColAgg->max = colVal.value.u64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_JSON: - break; - case TSDB_DATA_TYPE_VARBINARY: - break; - case TSDB_DATA_TYPE_DECIMAL: - break; - case TSDB_DATA_TYPE_BLOB: - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - break; - default: - ASSERT(0); - } + pColAgg->numOfNull++; } } } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index aa3f72d281..f773de5280 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { const STraceId *trace = &pMsg->info.traceId; if (!syncEnvIsStart()) { - vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_APP_ERROR; return -1; } SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); if (pSyncNode == NULL) { - vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } @@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } } @@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); @@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 38304bcc76..ee2aa5c3fa 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -420,7 +420,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray goto end; } } else { - metaGetTableTagsOpt(metaHandle, suid, uidList, tags); + metaGetTableTagsByUids(metaHandle, suid, uidList, tags); qInfo("succ to get table from meta idx, suid:%" PRIu64, suid); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 40fc8cb9a5..2251d09776 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock); +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -2415,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && nodeType(node->pRight) == QUERY_NODE_VALUE) { SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.type == TSDB_DATA_TYPE_NCHAR || pValue->node.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetStrValueFromNode(pValue); - strncpy(condTable, value, TSDB_TABLE_NAME_LEN); - taosMemoryFree(value); + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); return true; } } @@ -2480,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, condTableName); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smr, condTableName); - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByName(&smrChildTable, condTableName); + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); numOfRows = 0; } blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + doSetOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2508,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smr, suid); + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); if (numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); @@ -2562,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t blockDataCleanup(dataBlock); } -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock) { +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smr).me.name); + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); int32_t numOfRows = *pNumOfRows; - int32_t numOfTags = (*smr).me.stbEntry.schemaTag.nCols; + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; for (int32_t i = 0; i < numOfTags; ++i) { SColumnInfoData* pColInfoData = NULL; @@ -2588,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, // tag name char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smr).me.stbEntry.schemaTag.pSchema[i].name); + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); colDataAppend(pColInfoData, numOfRows, tagName, false); // tag type - int8_t tagType = (*smr).me.stbEntry.schemaTag.pSchema[i].type; + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); char tagTypeStr[VARSTR_HEADER_SIZE + 32]; int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); if (tagType == TSDB_DATA_TYPE_VARCHAR) { tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += - sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(tagTypeStr, tagTypeLen); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); STagVal tagVal = {0}; - tagVal.cid = (*smr).me.stbEntry.schemaTag.pSchema[i].colId; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; char* tagData = NULL; uint32_t tagLen = 0; if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; + tagData = (char*)smrChildTable->me.ctbEntry.pTags; } else { - bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); if (exist) { if (IS_VAR_DATA_TYPE(tagType)) { tagData = (char*)tagVal.pData; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f6fd8c8920..27b8e83a20 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -706,6 +706,9 @@ void* destroyStreamFillSupporter(SStreamFillSupporter* pFillSup) { pFillSup->pAllColInfo = destroyFillColumnInfo(pFillSup->pAllColInfo, pFillSup->numOfFillCols, pFillSup->numOfAllCols); tSimpleHashCleanup(pFillSup->pResMap); pFillSup->pResMap = NULL; + streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); + pFillSup->cur.pRowVal = NULL; + taosMemoryFree(pFillSup); return NULL; } @@ -722,6 +725,7 @@ void* destroyStreamFillInfo(SStreamFillInfo* pFillInfo) { taosMemoryFreeClear(pFillInfo->pResRow); } pFillInfo->pLinearInfo = destroyStreamFillLinearInfo(pFillInfo->pLinearInfo); + taosArrayDestroy(pFillInfo->delRanges); taosMemoryFree(pFillInfo); return NULL; } @@ -732,6 +736,8 @@ void destroyStreamFillOperatorInfo(void* param) { pInfo->pFillSup = destroyStreamFillSupporter(pInfo->pFillSup); pInfo->pRes = blockDataDestroy(pInfo->pRes); pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock); + pInfo->pPrevSrcBlock = blockDataDestroy(pInfo->pPrevSrcBlock); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); pInfo->pColMatchColInfo = taosArrayDestroy(pInfo->pColMatchColInfo); taosMemoryFree(pInfo); } @@ -743,6 +749,7 @@ static void resetFillWindow(SResultRowData* pRowData) { void resetPrevAndNextWindow(SStreamFillSupporter* pFillSup, SStreamState* pState) { resetFillWindow(&pFillSup->prev); + streamStateReleaseBuf(NULL, NULL, pFillSup->cur.pRowVal); resetFillWindow(&pFillSup->cur); resetFillWindow(&pFillSup->next); resetFillWindow(&pFillSup->nextNext); @@ -753,12 +760,12 @@ void getCurWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupI resetPrevAndNextWindow(pFillSup, pState); SWinKey key = {.ts = ts, .groupId = groupId}; - void* curVal = NULL; + // void* curVal = NULL; int32_t curVLen = 0; - int32_t code = streamStateFillGet(pState, &key, (void**)&curVal, &curVLen); + int32_t code = streamStateFillGet(pState, &key, (void**)&pFillSup->cur.pRowVal, &curVLen); ASSERT(code == TSDB_CODE_SUCCESS); pFillSup->cur.key = key.ts; - pFillSup->cur.pRowVal = curVal; + // pFillSup->cur.pRowVal = curVal; } void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamFillSupporter* pFillSup) { @@ -777,11 +784,9 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SWinKey preKey = {.groupId = groupId}; void* preVal = NULL; int32_t preVLen = 0; - if (pCur) { - code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); - } + code = streamStateGetGroupKVByCur(pCur, &preKey, (const void**)&preVal, &preVLen); - if (pCur && code == TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_SUCCESS) { pFillSup->prev.key = preKey.ts; pFillSup->prev.pRowVal = preVal; @@ -790,35 +795,36 @@ void getWindowFromDiscBuf(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, code = streamStateCurNext(pState, pCur); if (code != TSDB_CODE_SUCCESS) { + streamStateFreeCur(pCur); pCur = NULL; } } else { + streamStateFreeCur(pCur); pCur = streamStateFillSeekKeyNext(pState, &key); } - if (pCur) { - SWinKey nextKey = {.groupId = groupId}; - void* nextVal = NULL; - int32_t nextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->next.key = nextKey.ts; - pFillSup->next.pRowVal = nextVal; - if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { - code = streamStateCurNext(pState, pCur); + SWinKey nextKey = {.groupId = groupId}; + void* nextVal = NULL; + int32_t nextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextKey, (const void**)&nextVal, &nextVLen); + if (code == TSDB_CODE_SUCCESS) { + pFillSup->next.key = nextKey.ts; + pFillSup->next.pRowVal = nextVal; + if (pFillSup->type == TSDB_FILL_PREV || pFillSup->type == TSDB_FILL_NEXT) { + code = streamStateCurNext(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey nextNextKey = {.groupId = groupId}; + void* nextNextVal = NULL; + int32_t nextNextVLen = 0; + code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); if (code == TSDB_CODE_SUCCESS) { - SWinKey nextNextKey = {.groupId = groupId}; - void* nextNextVal = NULL; - int32_t nextNextVLen = 0; - code = streamStateGetGroupKVByCur(pCur, &nextNextKey, (const void**)&nextNextVal, &nextNextVLen); - if (code == TSDB_CODE_SUCCESS) { - pFillSup->nextNext.key = nextNextKey.ts; - pFillSup->nextNext.pRowVal = nextNextVal; - } + pFillSup->nextNext.key = nextNextKey.ts; + pFillSup->nextNext.pRowVal = nextNextVal; } } } } + streamStateFreeCur(pCur); } static bool hasPrevWindow(SStreamFillSupporter* pFillSup) { return pFillSup->prev.key != INT64_MIN; } @@ -1388,6 +1394,7 @@ static void doDeleteFillResult(SOperatorInfo* pOperator) { } pInfo->srcDelRowIndex++; } + streamStateFreeCur(pCur); doDeleteFillResultImpl(pOperator, ts, endTs, groupId); } pFillInfo->current = pFillInfo->end + 1; @@ -1538,9 +1545,12 @@ static int32_t initResultBuf(SStreamFillSupporter* pFillSup) { pFillSup->next.key = INT64_MIN; pFillSup->nextNext.key = INT64_MIN; pFillSup->prev.key = INT64_MIN; + pFillSup->cur.key = INT64_MIN; pFillSup->next.pRowVal = NULL; pFillSup->nextNext.pRowVal = NULL; pFillSup->prev.pRowVal = NULL; + pFillSup->cur.pRowVal = NULL; + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9b4b7f1ee8..8df6f15a1b 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -879,7 +879,12 @@ static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { SWinKey* pW = taosArrayGet(pWins, i); - taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + void* tmp = taosHashGet(pUpdatedMap, pW, sizeof(SWinKey)); + if (tmp) { + void* value = *(void**)tmp; + taosMemoryFree(value); + taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + } } } @@ -1402,7 +1407,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.skey <= endTsCols[i]) { + do { uint64_t winGpId = pGpDatas[i]; bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1410,10 +1415,15 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 taosArrayPush(pUpWins, &winRes); } if (pUpdatedMap) { - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + void* tmp = taosHashGet(pUpdatedMap, &winRes, sizeof(SWinKey)); + if (tmp) { + void* value = *(void**)tmp; + taosMemoryFree(value); + taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + } } getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } + } while (win.ekey <= endTsCols[i]); } } @@ -2872,12 +2882,13 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SAr pChildSup->rowEntryInfoOffset, &pChInfo->aggSup); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + releaseOutputBuf(pChInfo->pState, pWinRes, pChResult); } if (num > 0 && pUpdatedMap) { saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap); saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize); - releaseOutputBuf(pInfo->pState, pWinRes, pCurResult); } + releaseOutputBuf(pInfo->pState, pWinRes, pCurResult); } } @@ -2891,9 +2902,7 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; - void* pVal = NULL; - int32_t size = 0; - if (streamStateGet(pState, &key, &pVal, &size) == TSDB_CODE_SUCCESS) { + if (streamStateGet(pState, &key, NULL, 0) == TSDB_CODE_SUCCESS) { return false; } return true; @@ -3067,8 +3076,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + STimeWindow nextWin = {0}; + if (IS_FINAL_OP(pInfo)) { + nextWin = getFinalTimeWindow(ts, &pInfo->interval); + } else { + nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { @@ -3122,8 +3135,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + if (IS_FINAL_OP(pInfo)) { + forwardRows = 1; + } else { + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b65acc4672..6d81499dbc 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -468,6 +468,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); indexMultiTermQueryAdd(mtm, tm, qtype); ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); + indexMultiTermQueryDestroy(mtm); } else { bool reverse; FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); @@ -647,9 +648,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { - for (int i = 0; i < nParam; i++) sifFreeParam(¶ms[i]); output->status = SFLT_NOT_INDEX; - return code; + goto _return; } // ugly code, refactor later diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 2aa8345e03..5e787f9503 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -1025,6 +1025,7 @@ void fstDestroy(Fst* fst) { } bool fstGet(Fst* fst, FstSlice* b, Output* out) { + int ret = false; FstNode* root = fstGetRoot(fst); Output tOut = 0; int32_t len; @@ -1037,7 +1038,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { uint8_t inp = data[i]; Output res = 0; if (false == fstNodeFindInput(root, inp, &res)) { - return false; + goto _return; } FstTransition trn; @@ -1047,18 +1048,20 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { taosArrayPush(nodes, &root); } if (!FST_NODE_IS_FINAL(root)) { - return false; + goto _return; } else { tOut = tOut + FST_NODE_FINAL_OUTPUT(root); + ret = true; } +_return: for (int32_t i = 0; i < taosArrayGetSize(nodes); i++) { FstNode** node = (FstNode**)taosArrayGet(nodes, i); fstNodeDestroy(*node); } taosArrayDestroy(nodes); *out = tOut; - return true; + return ret; } FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx) { // refactor later @@ -1243,6 +1246,7 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; taosArrayPush(sws->stack, &s); + taosMemoryFree(trans); return true; } } diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 3a36010b42..8ce0ba1e69 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -139,6 +139,7 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r } } if (taosArrayGetSize(tinsts) == 0) { + taosArrayDestroy(tinsts); return false; } uint32_t *v = taosHashGet(builder->cache, &tinsts, sizeof(POINTER_BYTES)); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index f3009c051b..e746bb4303 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -267,10 +267,12 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); if (ret != 0) { + taosArrayDestroy(offsets); indexError("failed to find target tablelist"); return TSDB_CODE_TDB_FILE_CORRUPTED; } } + taosArrayDestroy(offsets); return 0; } static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { @@ -336,6 +338,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, } stmStDestroy(st); stmBuilderDestroy(sb); + taosArrayDestroy(offsets); return TSDB_CODE_SUCCESS; } static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { @@ -379,6 +382,7 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { ", size: %d, time cost: %" PRIu64 "us", tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost); } + taosMemoryFree(p); fstSliceDestroy(&key); return 0; } @@ -471,6 +475,9 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt } stmStDestroy(st); stmBuilderDestroy(sb); + taosArrayDestroy(offsets); + taosMemoryFree(p); + return TSDB_CODE_SUCCESS; } int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) { @@ -898,9 +905,8 @@ static int tfileReaderLoadFst(TFileReader* reader) { int64_t ts = taosGetTimestampUs(); int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset); int64_t cost = taosGetTimestampUs() - ts; - indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %" PRId64 ", time cost: %" PRId64 - "us", - nread, reader->header.fstOffset, fstSize, ctx->file.buf, size, cost); + indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread, + reader->header.fstOffset, fstSize, ctx->file.buf, size, cost); // we assuse fst size less than FST_MAX_SIZE assert(nread > 0 && nread <= fstSize); @@ -989,6 +995,7 @@ static SArray* tfileGetFileList(const char* path) { TdDirPtr pDir = taosOpenDir(path); if (NULL == pDir) { + taosArrayDestroy(files); return NULL; } TdDirEntryPtr pDirEntry; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 308afd467f..75b631ec9d 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ce90bd3da0..27adb79e3c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) { return false; } +static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pName) { + if (TSDB_DB_NAME_T == pName->type) { + return reserveDbVgInfoInCache(pName->acctId, pName->dbname, pCxt->pMetaCache); + } + return reserveTableVgroupInCacheExt(pName, pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) { + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt; + SName name = {0}; + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromInsTagsImpl(pCxt, &name); + } + return code; +} + static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, AUTH_TYPE authType) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); @@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) && + QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { + code = collectMetaKeyFromInsTags(pCxt); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9d805b8e33..882fa8950c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2181,7 +2181,11 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge } if (TSDB_DB_NAME_T == pTargetName->type) { - return getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); + int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); + if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } + return code; } SVgroupInfo vgInfo = {0}; @@ -2190,86 +2194,22 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge *pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo)); if (NULL == *pVgroupList) { code = TSDB_CODE_OUT_OF_MEMORY; + } else { + taosArrayPush(*pVgroupList, &vgInfo); } - } - if (TSDB_CODE_SUCCESS == code) { - taosArrayPush(*pVgroupList, &vgInfo); + } else if (TSDB_CODE_MND_DB_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; } return code; } -static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { - if (OP_TYPE_EQUAL != pOper->opType) { - return TSDB_CODE_SUCCESS; - } - - SColumnNode* pCol = NULL; - SValueNode* pVal = NULL; - if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { - pCol = (SColumnNode*)pOper->pLeft; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { - pVal = (SValueNode*)pOper->pLeft; - } - if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { - pCol = (SColumnNode*)pOper->pRight; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { - pVal = (SValueNode*)pOper->pRight; - } - if (NULL == pCol || NULL == pVal) { - return TSDB_CODE_SUCCESS; - } - - if (0 == strcmp(pCol->colName, "db_name")) { - return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); - } else if (0 == strcmp(pCol->colName, "table_name")) { - return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); - } - - return TSDB_CODE_SUCCESS; -} - -static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); - } -} - -static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { - if (LOGIC_COND_TYPE_AND != pCond->condType) { - return TSDB_CODE_SUCCESS; - } - - SNode* pNode = NULL; - FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } - if ('\0' == pName->dbname[0]) { - pName->type = 0; - } - return TSDB_CODE_SUCCESS; -} - -static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { - if (NULL == pWhere) { - return TSDB_CODE_SUCCESS; - } - - if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { - return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); - } - - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { - return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); - } - - return TSDB_CODE_SUCCESS; -} - static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { if (!isSelectStmt(pCxt->pCurrStmt)) { return TSDB_CODE_SUCCESS; } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SName targetName = {0}; - int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); if (TSDB_CODE_SUCCESS == code) { code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 3075be1a00..dab1f01574 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -420,6 +420,75 @@ end: return retCode; } +static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pOper, SName* pName) { + if (OP_TYPE_EQUAL != pOper->opType) { + return TSDB_CODE_SUCCESS; + } + + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + pVal = (SValueNode*)pOper->pRight; + } + if (NULL == pCol || NULL == pVal) { + return TSDB_CODE_SUCCESS; + } + + if (0 == strcmp(pCol->colName, "db_name")) { + return tNameSetDbName(pName, acctId, pVal->literal, strlen(pVal->literal)); + } else if (0 == strcmp(pCol->colName, "table_name")) { + return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); + } + + return TSDB_CODE_SUCCESS; +} + +static void getInsTagsTableTargetObjName(int32_t acctId, SNode* pNode, SName* pName) { + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pNode, pName); + } +} + +static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicConditionNode* pCond, SName* pName) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { getInsTagsTableTargetObjName(acctId, pNode, pName); } + if ('\0' == pName->dbname[0]) { + pName->type = 0; + } + return TSDB_CODE_SUCCESS; +} + +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { + if (NULL == pWhere) { + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { + int32_t code = getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pWhere, pName); + if (TSDB_CODE_SUCCESS == code && '\0' == pName->dbname[0]) { + pName->type = 0; + } + return code; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { + return getInsTagsTableTargetNameFromCond(acctId, (SLogicConditionNode*)pWhere, pName); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); } @@ -1173,7 +1242,7 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosArrayDestroy(p->pTableVgroupReq); p = taosHashIterate(pMetaCache->pInsertTables, p); - } + } taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 79a7038401..0027ac9ca1 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) { } TEST_F(ParserSelectTest, informationSchema) { - useDb("root", "test"); + useDb("root", "information_schema"); - run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); + run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); + + run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); } TEST_F(ParserSelectTest, withoutFrom) { diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 9e67249238..de2ce55459 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index df6e72ce46..4e013c44b8 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 1442ed2e05..63a28e17ed 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -32,7 +32,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } sprintf(streamPath, "%s/%s", pMeta->path, "checkpoints"); - mkdir(streamPath, 0755); + taosMulModeMkDir(streamPath, 0755); taosMemoryFree(streamPath); if (tdbTbOpen("task.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pTaskDb) < 0) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index da0d0fbd6d..596c16747b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -196,6 +196,7 @@ int32_t streamStateClear(SStreamState* pState) { SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); SWinKey delKey = {0}; int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); + streamStateFreeCur(pCur); if (code == 0) { streamStateDel(pState, &delKey); } else { @@ -225,6 +226,9 @@ int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void* int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) { // todo refactor + if (!pVal) { + return 0; + } streamFreeVal(pVal); return 0; } @@ -236,7 +240,7 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { int32_t c; SStateKey sKey = {.key = *key, .opNum = pState->number}; - tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); + tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c); if (c != 0) { taosMemoryFree(pCur); return NULL; @@ -253,7 +257,7 @@ SStreamStateCur* streamStateFillGetCur(SStreamState* pState, const SWinKey* key) int32_t c; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); if (c != 0) { - taosMemoryFree(pCur); + streamStateFreeCur(pCur); return NULL; } return pCur; @@ -266,6 +270,7 @@ SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { if (code == 0) { return pCur; } + streamStateFreeCur(pCur); } return NULL; } @@ -300,6 +305,9 @@ int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const vo } int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } uint64_t groupId = pKey->groupId; int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); if (code == 0) { @@ -360,7 +368,7 @@ SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); - if (pCur == NULL) { + if (!pCur) { return NULL; } if (tdbTbcOpen(pState->pFillStateDb, &pCur->pCur, NULL) < 0) { @@ -411,6 +419,9 @@ SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* } int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { + if (!pCur) { + return -1; + } // return tdbTbcMoveToNext(pCur->pCur); } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index cc8bdca75d..42aeeae19a 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -62,7 +62,7 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) { } memset(pDb->pgrHash, 0, tsize); - mkdir(dbname, 0755); + taosMulModeMkDir(dbname, 0755); #ifdef USE_MAINDB // open main db diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 191011111d..ff4fae3de8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1347,7 +1347,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { EPSET_FORWARD_INUSE(&pCtx->epSet); } else { if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet) < 0) { - tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn)); + tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn), pConn); } } addConnToPool(pThrd->pool, pConn); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ae9f3155b4..e86314d4bd 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -509,7 +509,7 @@ void transDQCancel(SDelayQueue* queue, SDelayTask* task) { if (heapSize(queue->heap) != 0) { HeapNode* minNode = heapMin(queue->heap); - if (minNode != NULL) return; + if (minNode == NULL) return; uint64_t now = taosGetTimestampMs(); SDelayTask* task = container_of(minNode, SDelayTask, node); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 70a47fe079..c63f8f398b 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -284,7 +284,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { } return; } else { - tError("%s conn %p read invalid packet, exceed limit, received from %s, local info:", transLabel(pTransInst), + tError("%s conn %p read invalid packet, exceed limit, received from %s, local info:%s", transLabel(pTransInst), conn, conn->dst, conn->src); destroyConn(conn, true); return; @@ -952,10 +952,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, #ifdef WINDOWS char pipeName[64]; - snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%p-" PRIu64, taosSafeRand(), GetCurrentProcessId()); + snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%d-%" PRIu64, taosSafeRand(), GetCurrentProcessId()); #else char pipeName[PATH_MAX] = {0}; - snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08X-" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(), + snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08d-%" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(), taosGetSelfPthreadId()); #endif ret = uv_pipe_bind(&srv->pipeListen, pipeName); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index cf74525543..0cc946ac21 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -121,6 +121,8 @@ int32_t taosMkDir(const char *dirname) { if (taosDirExist(dirname)) return 0; #ifdef WINDOWS int32_t code = _mkdir(dirname, 0755); +#elif defined(DARWIN) + int32_t code = mkdir(dirname, 0777); #else int32_t code = mkdir(dirname, 0755); #endif @@ -156,6 +158,8 @@ int32_t taosMulMkDir(const char *dirname) { *pos = '\0'; #ifdef WINDOWS code = _mkdir(temp, 0755); +#elif defined(DARWIN) + code = mkdir(dirname, 0777); #else code = mkdir(temp, 0755); #endif @@ -170,6 +174,8 @@ int32_t taosMulMkDir(const char *dirname) { if (*(pos - 1) != TD_DIRSEP[0]) { #ifdef WINDOWS code = _mkdir(temp, 0755); +#elif defined(DARWIN) + code = mkdir(dirname, 0777); #else code = mkdir(temp, 0755); #endif @@ -179,7 +185,6 @@ int32_t taosMulMkDir(const char *dirname) { } } - // int32_t code = mkdir(dirname, 0755); if (code < 0 && errno == EEXIST) { return 0; } @@ -215,6 +220,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { *pos = '\0'; #ifdef WINDOWS code = _mkdir(temp, mode); +#elif defined(DARWIN) + code = mkdir(dirname, 0777); #else code = mkdir(temp, mode); #endif @@ -229,6 +236,8 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { if (*(pos - 1) != TD_DIRSEP[0]) { #ifdef WINDOWS code = _mkdir(temp, mode); +#elif defined(DARWIN) + code = mkdir(dirname, 0777); #else code = mkdir(temp, mode); #endif diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 8f6800c7be..9511230f8d 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -59,34 +59,18 @@ void osDefaultInit() { if (tmpDir != NULL) { strcpy(tsTempDir, tmpDir); } - - if (configDir[0] == 0) { - strcpy(configDir, "C:\\TDengine\\cfg"); - } - strcpy(tsDataDir, "C:\\TDengine\\data"); - strcpy(tsLogDir, "C:\\TDengine\\log"); - strcpy(tsTempDir, "C:\\Windows\\Temp"); strcpy(tsOsName, "Windows"); - #elif defined(_TD_DARWIN_64) - if (configDir[0] == 0) { - strcpy(configDir, "/usr/local/etc/taos"); - } - strcpy(tsDataDir, "/usr/local/var/lib/taos"); - strcpy(tsLogDir, "/usr/local/var/log/taos"); - strcpy(tsTempDir, "/tmp/taosd"); strcpy(tsOsName, "Darwin"); - #else - if (configDir[0] == 0) { - strcpy(configDir, "/etc/taos"); - } - strcpy(tsDataDir, "/var/lib/taos"); - strcpy(tsLogDir, "/var/log/taos"); - strcpy(tsTempDir, "/tmp"); strcpy(tsOsName, "Linux"); - #endif + if (configDir[0] == 0) { + strcpy(configDir, TD_CFG_DIR_PATH); + } + strcpy(tsDataDir, TD_DATA_DIR_PATH); + strcpy(tsLogDir, TD_LOG_DIR_PATH); + strcpy(tsTempDir, TD_TMP_DIR_PATH); } void osUpdate() { diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index b34d071773..c1e02b48aa 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -801,7 +801,6 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { if (taosKeepTcpAlive(pSocket) < 0) { // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); - taosCloseSocket(&pSocket); return NULL; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 62b9d87628..be220c2585 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1; #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode #ifdef TD_TSZ bool lossyFloat = false; @@ -999,41 +999,212 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co /************************************************************************* * STREAM COMPRESSION *************************************************************************/ -#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -typedef struct SCompressor SCompressor; +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (a)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; } + +static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; } + static struct { int8_t type; int32_t bytes; int8_t isVarLen; + int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); + int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); + int64_t (*getI64)(const void *pData); + void (*putI64)(int64_t v, void *pData); } DATA_TYPE_INFO[] = { - {TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL - {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL - {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT - {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT - {TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT - {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT - {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT - {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE - {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR - {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP - {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR - {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT - {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT - {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT - {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT - {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON - {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY - {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL - {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB - {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB + {.type = TSDB_DATA_TYPE_NULL, + .bytes = 0, + .isVarLen = 0, + .startFn = NULL, + .cmprFn = NULL, + .endFn = NULL, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_BOOL, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompBoolStart, + .cmprFn = tCompBool, + .endFn = tCompBoolEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_TINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, + {.type = TSDB_DATA_TYPE_SMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, + {.type = TSDB_DATA_TYPE_INT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, + {.type = TSDB_DATA_TYPE_BIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, + {.type = TSDB_DATA_TYPE_FLOAT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompFloatStart, + .cmprFn = tCompFloat, + .endFn = tCompFloatEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_DOUBLE, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompDoubleStart, + .cmprFn = tCompDouble, + .endFn = tCompDoubleEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_VARCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_TIMESTAMP, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompTimestampStart, + .cmprFn = tCompTimestamp, + .endFn = tCompTimestampEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_NCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_UTINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, + {.type = TSDB_DATA_TYPE_USMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, + {.type = TSDB_DATA_TYPE_UINT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, + {.type = TSDB_DATA_TYPE_UBIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, + {.type = TSDB_DATA_TYPE_JSON, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_VARBINARY, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_BLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, + {.type = TSDB_DATA_TYPE_MEDIUMBLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd, + .getI64 = NULL, + .putI64 = NULL}, }; struct SCompressor { @@ -1041,8 +1212,9 @@ struct SCompressor { int8_t cmprAlg; int8_t autoAlloc; int32_t nVal; - uint8_t *aBuf[2]; - int64_t nBuf[2]; + uint8_t *pBuf; + int32_t nBuf; + uint8_t *aBuf[1]; union { // Timestamp ---- struct { @@ -1056,6 +1228,7 @@ struct SCompressor { int32_t i_selector; int32_t i_start; int32_t i_end; + int32_t i_nEle; uint64_t i_aZigzag[241]; int8_t i_aBitN[241]; }; @@ -1072,86 +1245,114 @@ struct SCompressor { }; }; -// Timestamp ===================================================== -static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { +static int32_t tTwoStageComp(SCompressor *pCmprsor, int32_t *szComp) { int32_t code = 0; - if (pCmprsor->nVal) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal); - if (code) return code; - } - pCmprsor->nBuf[1] = 0; - - int64_t n = 1; - int64_t value; - int64_t delta; - uint64_t vZigzag; - while (n < pCmprsor->nBuf[0]) { - uint8_t aN[2]; - aN[0] = pCmprsor->aBuf[0][n] & 0xf; - aN[1] = pCmprsor->aBuf[0][n] >> 4; - - n++; - - for (int32_t i = 0; i < 2; i++) { - vZigzag = 0; - for (uint8_t j = 0; j < aN[i]; j++) { - vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j)); - n++; - } - - int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); - if (pCmprsor->nBuf[1] == 0) { - delta = 0; - value = delta_of_delta; - } else { - delta = delta_of_delta + delta; - value = delta + value; - } - - memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t)); - pCmprsor->nBuf[1] += sizeof(int64_t); - - if (n >= pCmprsor->nBuf[0]) break; - } - } - - ASSERT(n == pCmprsor->nBuf[0]); - - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1); - if (code) return code; - } - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]); - pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1]; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; } - pCmprsor->aBuf[0][0] = 0; + *szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (*szComp && *szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *szComp += 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *szComp = pCmprsor->nBuf + 1; + } + + return code; +} + +// Timestamp ===================================================== +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 1; + + return code; +} + +static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(int64_t) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + int64_t value; + int64_t delta; + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t aN[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int32_t i = 0; i < 2; i++) { + uint64_t vZigzag = 0; + for (uint8_t j = 0; j < aN[i]; j++) { + vZigzag |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); + if (iVal) { + delta = delta_of_delta + delta; + value = delta + value; + } else { + delta = 0; + value = delta_of_delta; + } + + memcpy(pCmprsor->aBuf[0] + nBuf, &value, sizeof(value)); + nBuf += sizeof(int64_t); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + + ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nVal + 1); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 0; return code; } static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; int64_t ts = *(int64_t *)pData; - ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(nData == 8); - if (pCmprsor->aBuf[0][0] == 1) { + if (pCmprsor->pBuf[0] == 1) { if (pCmprsor->nVal == 0) { pCmprsor->ts_prev_val = ts; pCmprsor->ts_prev_delta = -ts; } if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } int64_t delta = ts - pCmprsor->ts_prev_val; if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } @@ -1162,43 +1363,63 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t pCmprsor->ts_prev_delta = delta; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0]; - pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p = pCmprsor->pBuf + pCmprsor->nBuf; + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] = 0; while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0]++; vZigzag >>= 8; } } else { while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] += 0x10; vZigzag >>= 8; } } } else { _copy_cmpr: - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts)); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(ts)))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts)); - pCmprsor->nBuf[0] += sizeof(ts); + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &ts, sizeof(ts)); + pCmprsor->nBuf += sizeof(ts); } pCmprsor->nVal++; return code; } +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) { + code = tCompTSSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; + *ppData = pCmprsor->aBuf[0]; + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + + return code; +} + // Integer ===================================================== #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; @@ -1207,94 +1428,163 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; +static const int32_t NEXT_IDX[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 0}; + +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + + pCmprsor->i_prev = 0; + pCmprsor->i_selector = 0; + pCmprsor->i_start = 0; + pCmprsor->i_end = 0; + pCmprsor->i_nEle = 0; + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + + return code; +} + +static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + int32_t size = DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], size))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + int64_t vPrev = 0; + while (n < pCmprsor->nBuf) { + uint64_t b; + memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); + n += sizeof(b); + + int32_t i_selector = (b & 0xf); + int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; + uint8_t bits = BIT_PER_INTEGER[i_selector]; + uint64_t mask = (((uint64_t)1) << bits) - 1; + for (int32_t iEle = 0; iEle < nEle; iEle++) { + uint64_t vZigzag = (b >> (bits * iEle + 4)) & mask; + vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; + + DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + } + } + + while (pCmprsor->i_nEle) { + vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev; + + memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; + } + + ASSERT(n == pCmprsor->nBuf && nBuf == size); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = size; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); - if (pCmprsor->aBuf[0][0] == 0) { - int64_t val; - - switch (pCmprsor->type) { - case TSDB_DATA_TYPE_TINYINT: - val = *(int8_t *)pData; - break; - case TSDB_DATA_TYPE_SMALLINT: - val = *(int16_t *)pData; - break; - case TSDB_DATA_TYPE_INT: - val = *(int32_t *)pData; - break; - case TSDB_DATA_TYPE_BIGINT: - val = *(int64_t *)pData; - break; - case TSDB_DATA_TYPE_UTINYINT: - val = *(uint8_t *)pData; - break; - case TSDB_DATA_TYPE_USMALLINT: - val = *(uint16_t *)pData; - break; - case TSDB_DATA_TYPE_UINT: - val = *(uint32_t *)pData; - break; - case TSDB_DATA_TYPE_UBIGINT: - val = *(int64_t *)pData; - break; - default: - ASSERT(0); - break; - } + if (pCmprsor->pBuf[0] == 0) { + int64_t val = DATA_TYPE_INFO[pCmprsor->type].getI64(pData); if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } int64_t diff = val - pCmprsor->i_prev; uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff); if (vZigzag >= SIMPLE8B_MAX) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0; pCmprsor->i_prev = val; - while (1) { - int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; - - if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { + for (;;) { + if (pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && + pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; } - pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; + pCmprsor->i_end = NEXT_IDX[pCmprsor->i_end]; + pCmprsor->i_nEle++; break; } else { - while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { - pCmprsor->i_selector++; - } - nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t)); - if (code) return code; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + } + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; } - uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]); - pCmprsor->nBuf[0] += sizeof(uint64_t); + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { - bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); - pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } // reset and continue pCmprsor->i_selector = 0; - for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) { + for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = NEXT_IDX[iVal]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) { pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]; } @@ -1303,18 +1593,146 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } } else { _copy_cmpr: - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData); if (code) return code; - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; return code; } +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + for (; pCmprsor->i_nEle;) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; + + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + } + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; + } + + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); + bp[0] = pCmprsor->i_selector; + uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; + for (int32_t iVal = 0; iVal < nEle; iVal++) { + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; + } + + pCmprsor->i_selector = 0; + } + + if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) { + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; + *ppData = pCmprsor->aBuf[0]; + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + + return code; +} + // Float ===================================================== +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + + pCmprsor->f_prev = 0; + pCmprsor->f_flag_p = NULL; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + + return code; +} + +static int32_t tCompFloatSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(float) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + float f; + uint32_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint32_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint32_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (32 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1347,13 +1765,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 9))) { + return code; } - pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->f_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1368,8 +1785,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1377,7 +1794,98 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat return code; } +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + if (pCmprsor->nBuf >= sizeof(float) * pCmprsor->nVal + 1) { + code = tCompFloatSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; + *ppData = pCmprsor->aBuf[0]; + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + + return code; +} + // Double ===================================================== +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + + pCmprsor->d_prev = 0; + pCmprsor->d_flag_p = NULL; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + + return code; +} + +static int32_t tCompDoubleSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(double) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + double f; + uint64_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint64_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (64 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1410,13 +1918,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->d_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1431,8 +1938,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1440,51 +1947,122 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa return code; } +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + if (pCmprsor->nBuf >= sizeof(double) * pCmprsor->nVal + 1) { + code = tCompDoubleSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; + *ppData = pCmprsor->aBuf[0]; + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + + return code; +} + // Binary ===================================================== +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + pCmprsor->nBuf = 1; + return 0; +} + static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; if (nData) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; return code; } +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + if (pCmprsor->nBuf == 1) return code; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf))) { + return code; + } + + int32_t szComp = + LZ4_compress_default(pCmprsor->pBuf + 1, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf - 1, pCmprsor->nBuf - 1); + if (szComp && szComp < pCmprsor->nBuf - 1) { + pCmprsor->aBuf[0][0] = 1; + *ppData = pCmprsor->aBuf[0]; + *nData = szComp + 1; + } else { + pCmprsor->pBuf[0] = 0; + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } + + return code; +} + // Bool ===================================================== static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + pCmprsor->nBuf = 0; + return 0; +} + static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; bool vBool = *(int8_t *)pData; - int32_t mod4 = pCmprsor->nVal & 3; + int32_t mod4 = (pCmprsor->nVal & 3); if (mod4 == 0) { - pCmprsor->nBuf[0]++; + pCmprsor->nBuf++; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf))) { + return code; } - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0; + pCmprsor->pBuf[pCmprsor->nBuf - 1] = 0; } if (vBool) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4]; + pCmprsor->pBuf[pCmprsor->nBuf - 1] |= BOOL_CMPR_TABLE[mod4]; } pCmprsor->nVal++; return code; } +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; + *ppData = pCmprsor->aBuf[0]; + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + + return code; +} + // SCompressor ===================================================== int32_t tCompressorCreate(SCompressor **ppCmprsor) { int32_t code = 0; @@ -1492,119 +2070,59 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) { *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor)); if ((*ppCmprsor) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + return code; } - code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024); - if (code) { - taosMemoryFree(*ppCmprsor); - *ppCmprsor = NULL; - goto _exit; - } - -_exit: return code; } int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t code = 0; - if (pCmprsor) { - int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); - for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { - tFree(pCmprsor->aBuf[iBuf]); - } + tFree(pCmprsor->pBuf); - taosMemoryFree(pCmprsor); + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); + for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { + tFree(pCmprsor->aBuf[iBuf]); } + taosMemoryFree(pCmprsor); + return code; } -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) { +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; pCmprsor->cmprAlg = cmprAlg; - pCmprsor->autoAlloc = autoAlloc; + pCmprsor->autoAlloc = 1; pCmprsor->nVal = 0; - switch (type) { - case TSDB_DATA_TYPE_TIMESTAMP: - pCmprsor->ts_prev_val = 0; - pCmprsor->ts_prev_delta = 0; - pCmprsor->ts_flag_p = NULL; - pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_BOOL: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_BINARY: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - pCmprsor->f_prev = 0; - pCmprsor->f_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_DOUBLE: - pCmprsor->d_prev = 0; - pCmprsor->d_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - pCmprsor->i_prev = 0; - pCmprsor->i_selector = 0; - pCmprsor->i_start = 0; - pCmprsor->i_end = 0; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - default: - break; + if (DATA_TYPE_INFO[type].startFn) { + DATA_TYPE_INFO[type].startFn(pCmprsor, type, cmprAlg); } return code; } -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) { +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin) { int32_t code = 0; - if (pCmprsor->nVal == 0) { - *ppData = NULL; - *nData = 0; - return code; + *ppOut = NULL; + *nOut = 0; + if (nOrigin) { + if (DATA_TYPE_INFO[pCmprsor->type].isVarLen) { + *nOrigin = pCmprsor->nBuf - 1; + } else { + *nOrigin = pCmprsor->nVal * DATA_TYPE_INFO[pCmprsor->type].bytes; + } } - if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) { - code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1); - if (code) return code; + if (pCmprsor->nVal == 0) return code; - int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]); - if (ret) { - pCmprsor->aBuf[1][0] = 0; - pCmprsor->nBuf[1] = ret + 1; - } else { - pCmprsor->aBuf[1][0] = 1; - memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1; - } - - *ppData = pCmprsor->aBuf[1]; - *nData = pCmprsor->nBuf[1]; - } else { - *ppData = pCmprsor->aBuf[0]; - *nData = pCmprsor->nBuf[0]; + if (DATA_TYPE_INFO[pCmprsor->type].endFn) { + return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppOut, nOut); } return code; @@ -1612,4 +2130,274 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); -} \ No newline at end of file +} + +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ +// Timestamp ===================================================== +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressTimestampImp(pBuf, nEle, pOut); + } else { + assert(0); + return -1; + } +} + +// Float ===================================================== +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { +#ifdef TD_TSZ + // lossy mode + if (lossyFloat) { + return tsCompressFloatLossyImp(pIn, nEle, pOut); + // lossless mode + } else { +#endif + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressFloatImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { +#ifdef TD_TSZ + if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); + } else { +#endif + // decompress lossless + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressFloatImp(pBuf, nEle, pOut); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Double ===================================================== +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { +#ifdef TD_TSZ + if (lossyDouble) { + // lossy mode + return tsCompressDoubleLossyImp(pIn, nEle, pOut); + } else { +#endif + // lossless mode + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); + } else { +#endif + // decompress lossless + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressDoubleImp(pBuf, nEle, pOut); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Binary ===================================================== +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsCompressStringImp(pIn, nIn, pOut, nOut); +} + +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsDecompressStringImp(pIn, nIn, pOut, nOut); +} + +// Bool ===================================================== +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressBoolImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressBoolImp(pBuf, nEle, pOut); + } else { + assert(0); + return -1; + } +} + +// Tinyint ===================================================== +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else { + assert(0); + return -1; + } +} + +// Smallint ===================================================== +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else { + assert(0); + return -1; + } +} + +// Int ===================================================== +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT); + } else { + assert(0); + return -1; + } +} + +// Bigint ===================================================== +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else { + assert(0); + return -1; + } +} diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index 18b22b51ce..0a835ec564 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -19,10 +19,9 @@ from util.dnodes import * class TDTestCase: def caseDescription(self): - ''' + """ [TD-11510] taosBenchmark test cases - ''' - return + """ def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -31,19 +30,19 @@ class TDTestCase: def getPath(self, tool="taosBenchmark"): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] paths = [] for root, dirs, files in os.walk(projPath): - if ((tool) in files): + if (tool) in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): + if "packaging" not in rootRealPath: paths.append(os.path.join(root, tool)) break - if (len(paths) == 0): + if len(paths) == 0: tdLog.exit("taosBenchmark not found!") return else: @@ -52,14 +51,15 @@ class TDTestCase: def run(self): binPath = self.getPath() - cmd = "%s -f ./5-taos-tools/taosbenchmark/json/default.json" %binPath + cmd = "%s -f ./5-taos-tools/taosbenchmark/json/default.json" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("reset query cache") tdSql.query("select count(*) from (select distinct(tbname) from db.stb)") tdSql.checkData(0, 0, 10) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 100) + if len(tdSql.queryResult): + tdLog.exit("query result is %d" % len(tdSql.queryResult)) def stop(self): tdSql.close() diff --git a/tests/script/tsim/stream/distributeIntervalRetrive0.sim b/tests/script/tsim/stream/distributeIntervalRetrive0.sim index 79edea2a3c..bea70b1639 100644 --- a/tests/script/tsim/stream/distributeIntervalRetrive0.sim +++ b/tests/script/tsim/stream/distributeIntervalRetrive0.sim @@ -3,12 +3,15 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v + sleep 50 sql connect sql create dnode $hostname2 port 7200 system sh/exec.sh -n dnode2 -s start +#==system sh/exec.sh -n dnode2 -s start -v print ===== step1 $x = 0 @@ -232,4 +235,41 @@ endi print loop3 over + + + + +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi + + + +#==system sh/exec.sh -n dnode2 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode2 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 + + + system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index 8e6972975e..e14062e830 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -4,6 +4,8 @@ looptest: system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v + sleep 200 sql connect @@ -353,6 +355,22 @@ endi +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 + + sql drop stream if exists streams0; diff --git a/tests/script/tsim/stream/fillIntervalLinear.sim b/tests/script/tsim/stream/fillIntervalLinear.sim index 46ff785fd3..4d0f101b6c 100644 --- a/tests/script/tsim/stream/fillIntervalLinear.sim +++ b/tests/script/tsim/stream/fillIntervalLinear.sim @@ -4,6 +4,8 @@ looptest: system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v + sleep 200 sql connect @@ -671,6 +673,22 @@ endi + + +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 sql drop stream if exists streams0; diff --git a/tests/script/tsim/stream/fillIntervalPrevNext.sim b/tests/script/tsim/stream/fillIntervalPrevNext.sim index 5eab5fdac1..4eadd7e7b1 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext.sim @@ -4,6 +4,8 @@ looptest: system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v + sleep 200 sql connect @@ -1011,6 +1013,21 @@ endi + +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 diff --git a/tests/script/tsim/stream/fillIntervalValue.sim b/tests/script/tsim/stream/fillIntervalValue.sim index 113eae9270..49e68ae9f2 100644 --- a/tests/script/tsim/stream/fillIntervalValue.sim +++ b/tests/script/tsim/stream/fillIntervalValue.sim @@ -4,6 +4,7 @@ looptest: system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v sleep 200 sql connect @@ -463,6 +464,20 @@ endi +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index bd8d3b0579..b7477fe36c 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -369,9 +369,10 @@ endi #$loop_all = 0 #=looptest: -sql drop database IF EXISTS test2; sql drop stream IF EXISTS streams21; sql drop stream IF EXISTS streams22; +sql drop stream IF EXISTS streams23; +sql drop database IF EXISTS test2; sql create database test2 vgroups 6; sql use test2; @@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2); sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -508,6 +510,50 @@ if $data32 != 8 then goto loop3 endi + +$loop_count = 0 + +loop4: +sleep 100 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt3; + +# row 0 +if $rows != 5 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 4 then + print =====data31=$data31 + goto loop4 +endi + +if $data41 != 2 then + print =====data41=$data41 + goto loop4 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 1902945bf6..bcc6725848 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -64,7 +64,7 @@ class TDTestCase: ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) - tdLog.info("restart taosd to ensure that the data falls into the disk") + tdLog.info("flush database to ensure that the data falls into the disk") # tdDnodes.stop(1) # tdDnodes.start(1) tdSql.query("flush database %s"%(paraDict['dbName'])) @@ -87,7 +87,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 10, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 03097e31b9..98d21a4f2c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,6 +1,11 @@ IF (TD_WEBSOCKET) + IF (TD_LINUX) + SET(websocket_lib_file "libtaosws.so") + ELSEIF (TD_DARWIN) + SET(websocket_lib_file "libtaosws.dylib") + ENDIF () MESSAGE("${Green} use libtaos-ws${ColourReset}") - IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/libtaosws.so" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") + IF (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/${websocket_lib_file}" OR "${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs/target/release/${websocket_lib_file}" IS_NEWER_THAN "${CMAKE_SOURCE_DIR}/.git/modules/tools/taosws-rs/FETCH_HEAD") include(ExternalProject) ExternalProject_Add(taosws-rs PREFIX "taosws-rs" @@ -15,7 +20,7 @@ IF (TD_WEBSOCKET) COMMAND cargo build --release -p taos-ws-sys COMMAND ./taos-ws-sys/ci/package.sh INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E copy target/libtaosws/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include ) @@ -34,7 +39,7 @@ IF (TD_WEBSOCKET) COMMAND cargo build --release -p taos-ws-sys COMMAND ./taos-ws-sys/ci/package.sh INSTALL_COMMAND - COMMAND cmake -E copy target/libtaosws/libtaosws.so ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E copy target/libtaosws/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include COMMAND cmake -E copy target/libtaosws/taosws.h ${CMAKE_BINARY_DIR}/build/include ) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 488b623f89..45a6f8c16f 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -6,6 +6,10 @@ IF (TD_LINUX AND TD_WEBSOCKET) ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include -ltaosws) SET(LINK_WEBSOCKET "-L${CMAKE_BINARY_DIR}/build/lib -ltaosws") ADD_DEPENDENCIES(shell taosws-rs) +ELSEIF (TD_DARWIN AND TD_WEBSOCKET) + ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include) + SET(LINK_WEBSOCKET "${CMAKE_BINARY_DIR}/build/lib/libtaosws.dylib") + ADD_DEPENDENCIES(shell taosws-rs) ELSE () SET(LINK_WEBSOCKET "") ENDIF ()