xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into merge/mainto3.0

This commit is contained in:
Shengliang Guan 2024-10-30 18:04:58 +08:00
parent 4a412e0814 dba6656a38
commit 91a352a329
209 changed files with 18000 additions and 1465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/taoskeeper-ci.yml vendored Normal file
View File

@ -0,0 +1,58 @@
name: TaosKeeper CI
on:
push:
paths:
- tools/keeper/**
jobs:
build:
runs-on: ubuntu-latest
name: Run unit tests
steps:
- name: Checkout the repository
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.18
- name: Install system dependencies
run: |
sudo apt update -y
sudo apt install -y build-essential cmake libgeos-dev
- name: Install TDengine
run: |
mkdir debug
cd debug
cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true
make -j 4
sudo make install
which taosd
which taosadapter
which taoskeeper
- name: Start taosd
run: |
cp /etc/taos/taos.cfg ./
sudo echo "supportVnodes 256" >> taos.cfg
nohup sudo taosd -c taos.cfg &
- name: Start taosadapter
run: nohup sudo taosadapter &
- name: Run tests with coverage
working-directory: tools/keeper
run: |
go mod tidy
go test -v -coverpkg=./... -coverprofile=coverage.out ./...
go tool cover -func=coverage.out
- name: Clean up
if: always()
run: |
if pgrep taosd; then sudo pkill taosd; fi
if pgrep taosadapter; then sudo pkill taosadapter; fi

14
cmake/cmake.define
View File

@ -1,6 +1,7 @@
cmake_minimum_required(VERSION 3.0) cmake_minimum_required(VERSION 3.0)
set(CMAKE_VERBOSE_MAKEFILE FALSE) set(CMAKE_VERBOSE_MAKEFILE FALSE)
set(TD_BUILD_TAOSA_INTERNAL FALSE) set(TD_BUILD_TAOSA_INTERNAL FALSE)
set(TD_BUILD_KEEPER_INTERNAL FALSE)
# set output directory # set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
@ -57,6 +58,19 @@ IF(TD_BUILD_HTTP)
ADD_DEFINITIONS(-DHTTP_EMBEDDED) ADD_DEFINITIONS(-DHTTP_EMBEDDED)
ENDIF() ENDIF()
IF("${BUILD_KEEPER}" STREQUAL "")
SET(TD_BUILD_KEEPER FALSE)
ELSEIF(${BUILD_KEEPER} MATCHES "false")
SET(TD_BUILD_KEEPER FALSE)
ELSEIF(${BUILD_KEEPER} MATCHES "true")
SET(TD_BUILD_KEEPER TRUE)
ELSEIF(${BUILD_KEEPER} MATCHES "internal")
SET(TD_BUILD_KEEPER FALSE)
SET(TD_BUILD_KEEPER_INTERNAL TRUE)
ELSE()
SET(TD_BUILD_KEEPER FALSE)
ENDIF()
IF("${BUILD_TOOLS}" STREQUAL "") IF("${BUILD_TOOLS}" STREQUAL "")
IF(TD_LINUX) IF(TD_LINUX)
IF(TD_ARM_32) IF(TD_ARM_32)

10
docs/zh/06-advanced/06-data-analysis/01-arima.md Normal file
View File

@ -0,0 +1,10 @@
---
title: "ARIMA"
sidebar_label: "ARIMA"
---
本节讲述如何 ARIMA 算法的使用方法。
## 功能概述
……

13
docs/zh/06-advanced/06-data-analysis/index.md Normal file
View File

@ -0,0 +1,13 @@
---
sidebar_label: 数据分析
title: 数据分析功能
---
## 概述
TDengine 提供数据分析功能的扩展组件，通过引入 ANode，TDengine 能够支持时间序列的机器学习分析
下图展示了数据分析的技术架构。
![数据分析功能架构图](./pic/data-analysis.png)

BIN
docs/zh/06-advanced/06-data-analysis/pic/data-analysis.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

18
docs/zh/08-operation/02-planning.md
View File

@ -53,7 +53,7 @@ M = (T × S × 3 + (N / 4096) + 100)
与 WebSocket 连接方式相比，RESTful 连接方式在内存占用上更大，除了缓冲区所需的内存以外，还需要考虑每个连接响应结果的内存开销。这种内存开销与响应结果的JSON 数据大小密切相关，特别是在查询数据量很大时，会占用大量内存。 与 WebSocket 连接方式相比，RESTful 连接方式在内存占用上更大，除了缓冲区所需的内存以外，还需要考虑每个连接响应结果的内存开销。这种内存开销与响应结果的JSON 数据大小密切相关，特别是在查询数据量很大时，会占用大量内存。
由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议打开batchfetch=true 选项，以启用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险 由于 RESTful 连接方式不支持分批获取查询数据，这就导致在查询获取超大结果集时，可能会占用特别大的内存，从而导致内存溢出，因此，在大型项目中，建议使用 WebSocket 连接方式，实现流式结果集返回，从而避免内存溢出的风险
**注意** **注意**
- 建议采用 RESTful/WebSocket 连接方式来访问 TDengine 集群，而不采用taosc 原生连接方式。 - 建议采用 RESTful/WebSocket 连接方式来访问 TDengine 集群，而不采用taosc 原生连接方式。
@ -146,11 +146,11 @@ TDengine 的多级存储功能在使用上还具备以下优点。
下表列出了 TDengine 的一些接口或组件的常用端口，这些端口均可以通过配置文件中的参数进行修改。 下表列出了 TDengine 的一些接口或组件的常用端口，这些端口均可以通过配置文件中的参数进行修改。
|接口或组件 | 端口 | | 接口或组件 | 端口 |
|:---------------------------:|:---------:| | :---------------: | :--------: |
|原生接口（taosc） | 6030 | | 原生接口（taosc） | 6030 |
|RESTful 接口 | 6041 | | RESTful 接口 | 6041 |
|WebSocket 接口 |6041 | | WebSocket 接口 | 6041 |
|taosKeeper | 6043 | | taosKeeper | 6043 |
|taosX | 6050, 6055 | | taosX | 6050, 6055 |
|taosExplorer | 6060 | | taosExplorer | 6060 |

36
docs/zh/08-operation/12-multi.md
View File

@ -76,7 +76,7 @@ dataDir /mnt/data6 2 0
|s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 | |s3UploadDelaySec | data 文件持续多长时间不再变动后上传至 s3，单位：秒。最小值：1；最大值：2592000 (30天），默认值 60 秒 |
|s3PageCacheSize |s3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024\*1024。 ，默认值 4096| |s3PageCacheSize |s3 page cache 缓存页数目，单位：页。最小值：4；最大值：1024*1024\*1024。 ，默认值 4096|
|s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 | |s3MigrateIntervalSec | 本地数据文件自动上传 S3 的触发周期，单位为秒。最小值：600；最大值：100000。默认值 3600 |
|s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 1，表示开启自动 S3 迁移，可配置为 0。 | |s3MigrateEnabled | 是否自动进行 S3 迁移，默认值为 0，表示关闭自动 S3 迁移，可配置为 1。 |
### 检查配置参数可用性 ### 检查配置参数可用性
@ -108,9 +108,37 @@ s3migrate database <db_name>;
| # | 参数 | 默认值 | 最小值 | 最大值 | 描述 | | # | 参数 | 默认值 | 最小值 | 最大值 | 描述 |
| :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- | | :--- | :----------- | :----- | :----- | :------ | :----------------------------------------------------------- |
| 1 | s3_keeplocal | 3650 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 | | 1 | s3_keeplocal | 365 | 1 | 365000 | 数据在本地保留的天数，即 data 文件在本地磁盘保留多长时间后可以上传到 S3。默认单位：天，支持 m（分钟）、h（小时）和 d（天）三个单位 |
| 2 | s3_chunksize | 262144 | 131072 | 1048576 | 上传对象的大小阈值，与 TSDB_PAGESIZE 参数一样，不可修改，单位为 TSDB 页 | | 2 | s3_chunkpages | 131072 | 131072 | 1048576 | 上传对象的大小阈值，与 tsdb_pagesize 参数一样，不可修改，单位为 TSDB 页 |
| 3 | s3_compact | 0 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。 | | 3 | s3_compact | 1 | 0 | 1 | TSDB 文件组首次上传 S3 时，是否自动进行 compact 操作。 |
### 对象存储读写次数估算
对象存储服务的使用成本与存储的数据量及请求次数相关，下面分别介绍数据的上传及下载过程。
#### 数据上传
当 TSDB 时序数据超过 `s3_keeplocal` 参数指定的时间，相关的数据文件会被切分成多个文件块，每个文件块的默认大小是 512M 字节 (`s3_chunkpages * tsdb_pagesize`)。除了最后一个文件块保留在本地文件系统外，其余的文件块会被上传到对象存储服务。
```math
上传次数 = 数据文件大小 / (s3_chunkpages * tsdb_pagesize) - 1
```
在创建数据库时，可以通过 `s3_chunkpages` 参数调整每个文件块的大小，从而控制每个数据文件的上传次数。
其它类型的文件如 head, stt, sma 等，保留在本地文件系统，以加速预计算相关查询。
#### 数据下载
在查询操作中，如果需要访问对象存储中的数据，TSDB 不会下载整个数据文件，而是计算所需数据在文件中的位置，只下载相应的数据到 TSDB 页缓存中，然后将数据返回给查询执行引擎。后续查询首先检查页缓存，查看数据是否已被缓存。如果数据已缓存，则直接使用缓存中的数据，而无需重复从对象存储下载，从而有效降低从对象存储下载数据的次数。
相邻的多个数据页会作为一个数据块从对象存储下载一次，以减少从对象存储下载的次数。每个数据页的大小，在创建数据库时，通过 `tsdb_pagesize` 参数指定，默认 4K 字节。
```math
下载次数 = 查询需要的数据块数量 - 已缓存的数据块数量
```
页缓存是内存缓存，节点重启后，再次查询需要重新下载数据。缓存采用 LRU (Least Recently Used) 策略，当缓存空间不足时，最近最少使用的数据将被淘汰。缓存的大小可以通过 `s3PageCacheSize` 参数进行调整，通常来说，缓存越大，下载次数越少。
## Azure Blob 存储 ## Azure Blob 存储
本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。 本节介绍在 TDengine Enterprise 如何使用微软 Azure Blob 对象存储。本功能是上一小节‘对象存储’功能的扩展，需额外依赖 Flexify 服务提供的 S3 网关。通过适当的参数配置，可以把大部分较冷的时序数据存储到 Azure Blob 服务中。

23
docs/zh/14-reference/05-connector/50-odbc.mdx
View File

@ -3,11 +3,14 @@ sidebar_label: ODBC
title: TDengine ODBC title: TDengine ODBC
--- ---
TDengine ODBC 是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库的数据表/视图。 TDengine ODBC 是为 TDengine 实现的 ODBC 驱动程序，支持 Windows 系统的应用（如 [PowerBI](https://powerbi.microsoft.com/zh-cn/) 等）以及用户自定义开发的应用程序，通过 ODBC 标准接口访问本地、远程和云服务的 TDengine 数据库。
TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连接 TDengine 数据库，使用时可以为 TDengine 数据源设置不同的连接方式。访问云服务时必须使用 WebSocket 连接方式。 TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连接 TDengine 数据库，使用时可以为 TDengine 数据源设置不同的连接方式。访问云服务时必须使用 WebSocket 连接方式。
注意：TDengine ODBC 支持 32/64 位 Windows 系统，调用 TDengine ODBC 需要通过相应位数的 ODBC 驱动管理器进行。在 32 位 Windows 系统或者 64 位 Windows 系统的 32 位应用程序中，仅支持使用 WebSocket 连接方式访问 TDengine 数据库。 TDengine ODBC 提供 64 位 和 32 位 两种驱动程序。但 32 位仅企业版支持，且仅支持 WebSocket 连接方式。
**注意：**
- 驱动管理器：确保使用与应用程序架构匹配的 ODBC 驱动管理器。32 位应用程序需要使用 32 位 ODBC 驱动管理器，64 位应用程序需要使用 64 位 ODBC 驱动管理器。
- 数据源名称（DSN）：32 位和 64 位 ODBC 驱动管理器都可以看到所有 DSN，用户 DSN 标签页下的 DSN 如果名字相同会共用，因此需要在 DSN 名称上去区分。
想更多了解 TDengine 时序时序数据库的使用，可访问 [TDengine官方文档](https://docs.taosdata.com/intro/)。 想更多了解 TDengine 时序时序数据库的使用，可访问 [TDengine官方文档](https://docs.taosdata.com/intro/)。
@ -24,17 +27,17 @@ TDengine ODBC 提供基于 WebSocket（推荐）和 原生连接两种方式连
### 数据源连接类型与区别 ### 数据源连接类型与区别
TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与 Native 连接，其区别如下： TDengine ODBC 支持两种连接 TDengine 数据库方式：WebSocket 连接与 Native 连接，其区别如下：
1. 访问云服务仅支持使用 Websocket 连接方式。 1. 访问云服务仅支持使用 WebSocket 连接方式。
2. 32 位应用程序仅支持使用 WebSocket 连接方式。 2. 32 位应用程序仅支持使用 WebSocket 连接方式。
3. Websocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。 3. WebSocket 连接的兼容性更好，一般不需要随着 TDengine 数据库服务端升级而升级客户端的库。
4. Native 连接通常性能更好一点，但是必须与 TDengine 数据库服务端的版本保持一致。 4. Native 连接通常性能更好一点，但是必须与 TDengine 数据库服务端的版本保持一致。
5. 对于一般用户，建议使用 **Websocket** 连接方式，性能与 Native 差别不大，兼容性更好。 5. 对于一般用户，建议使用 **WebSocket** 连接方式，性能与 Native 差别不大，兼容性更好。
### WebSocket 连接 ### WebSocket 连接
@ -46,11 +49,11 @@ TDengine ODBC 支持两种连接 TDengine 数据库方式：Websocket 连接与
4. 点击完成，进入 TDengine ODBC 数据源配置页面，填写如下必要信息 4. 点击完成，进入 TDengine ODBC 数据源配置页面，填写如下必要信息
![ODBC websocket connection config](./assets/odbc-ws-config-zh.webp) ![ODBC WebSocket connection config](./assets/odbc-ws-config-zh.webp)
4.1 【DSN】:Data Source Name 必填，为新添加的 ODBC 数据源命名 4.1 【DSN】:Data Source Name 必填，为新添加的 ODBC 数据源命名
4.2【连接类型】 : 必选，选择连接类型，这里选择 【Websocket】 4.2【连接类型】 : 必选，选择连接类型，这里选择 【WebSocket】
4.3【URL】必填，ODBC 数据源 URL，示例: `http://localhost:6041`， 云服务的 url 示例: `https://gw.cloud.taosdata.com?token=your_token` 4.3【URL】必填，ODBC 数据源 URL，示例: `http://localhost:6041`， 云服务的 url 示例: `https://gw.cloud.taosdata.com?token=your_token`
@ -111,7 +114,7 @@ WebSocket 连接方式除此之外还支持 Windows X64系统上运行的 32 位
| taos_odbc 版本 | 主要变化 | TDengine 版本 | | taos_odbc 版本 | 主要变化 | TDengine 版本 |
| :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- | | :----------- | :-------------------------------------------------------------------------------------------------- | :---------------- |
| v1.1.0 | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型； | 3.3.3.0及更高版本 | | v1.1.0 | 1. 支持视图功能；<br/>2. 支持 VARBINARY/GEOMETRY 数据类型；<br/>3. 支持 ODBC 32 位 WebSocket 连接方式（仅企业版支持）；<br/>4. 支持 ODBC 数据源配置对话框设置对工业软件 KingSCADA、Kepware 等的兼容性适配选项（仅企业版支持）； | 3.3.3.0及更高版本 |
| v1.0.2 | 支持 CP1252 字符编码； | 3.2.3.0及更高版本 | | v1.0.2 | 支持 CP1252 字符编码； | 3.2.3.0及更高版本 |
| v1.0.1 | 1. 支持 DSN 设置 BI 模式，在 BI 模式下 TDengine 数据库不返回系统数据库和超级表子表信息；<br/>2. 重构字符集转换模块，提升读写性能；<br/>3. ODBC 数据源配置对话框中默认修改默认连接方式为“WebSocket”；<br/>4. ODBC 数据源配置对话框增加“测试连接”控件；<br/>5. ODBC 数据源配置支持中文/英文界面； | - | | v1.0.1 | 1. 支持 DSN 设置 BI 模式，在 BI 模式下 TDengine 数据库不返回系统数据库和超级表子表信息；<br/>2. 重构字符集转换模块，提升读写性能；<br/>3. ODBC 数据源配置对话框中默认修改默认连接方式为“WebSocket”；<br/>4. ODBC 数据源配置对话框增加“测试连接”控件；<br/>5. ODBC 数据源配置支持中文/英文界面； | - |
| v1.0.0.0 | 发布初始版本，支持与Tdengine数据库交互以读写数据，具体请参考“API 参考”一节 | 3.2.2.0及更高版本 | | v1.0.0.0 | 发布初始版本，支持与Tdengine数据库交互以读写数据，具体请参考“API 参考”一节 | 3.2.2.0及更高版本 |

4
docs/zh/14-reference/07-supported.md
View File

@ -39,10 +39,10 @@ description: "TDengine 服务端、客户端和连接器支持的平台列表"
| ----------- | ------------- | ------------- | ------------- | --------- | --------- | | ----------- | ------------- | ------------- | ------------- | --------- | --------- |
| **OS** | **Linux** | **Win64** | **macOS** | **Linux** | **macOS** | | **OS** | **Linux** | **Win64** | **macOS** | **Linux** | **macOS** |
| **C/C++** | ● | ● | ● | ● | ● | | **C/C++** | ● | ● | ● | ● | ● |
| **JDBC** | ● | ● | ○ | ● | ○ | | **JDBC** | ● | ● | ● | ● | ● |
| **Python** | ● | ● | ● | ● | ● | | **Python** | ● | ● | ● | ● | ● |
| **Go** | ● | ● | ● | ● | ● | | **Go** | ● | ● | ● | ● | ● |
| **NodeJs** | ● | ● | ○ | ● | ○ | | **NodeJs** | ● | ● | ● | ● | ● |
| **C#** | ● | ● | ○ | ● | ○ | | **C#** | ● | ● | ○ | ● | ○ |
| **Rust** | ● | ● | ● | ○ | ● | | **Rust** | ● | ● | ● | ○ | ● |
| **RESTful** | ● | ● | ● | ● | ● | | **RESTful** | ● | ● | ● | ● | ● |

22
include/common/tmsg.h
View File

@ -4104,18 +4104,16 @@ void tDeleteMqMetaRsp(SMqMetaRsp* pRsp);
#define MQ_DATA_RSP_VERSION 100 #define MQ_DATA_RSP_VERSION 100
typedef struct { typedef struct {
struct { SMqRspHead head;
SMqRspHead head; STqOffsetVal rspOffset;
STqOffsetVal rspOffset; STqOffsetVal reqOffset;
STqOffsetVal reqOffset; int32_t blockNum;
int32_t blockNum; int8_t withTbName;
int8_t withTbName; int8_t withSchema;
int8_t withSchema; SArray* blockDataLen;
SArray* blockDataLen; SArray* blockData;
SArray* blockData; SArray* blockTbName;
SArray* blockTbName; SArray* blockSchema;
SArray* blockSchema;
};
union{ union{
struct{ struct{

776
include/common/ttokendef.h
View File

@ -16,394 +16,394 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_NK_COMMA 33 #define TK_NK_COMMA 33
#define TK_HOST 34 #define TK_HOST 34
#define TK_IS_IMPORT 35 #define TK_IS_IMPORT 35
#define TK_NK_INTEGER 36 #define TK_NK_INTEGER 36
#define TK_CREATEDB 37 #define TK_CREATEDB 37
#define TK_USER 38 #define TK_USER 38
#define TK_ENABLE 39 #define TK_ENABLE 39
#define TK_SYSINFO 40 #define TK_SYSINFO 40
#define TK_ADD 41 #define TK_ADD 41
#define TK_DROP 42 #define TK_DROP 42
#define TK_GRANT 43 #define TK_GRANT 43
#define TK_ON 44 #define TK_ON 44
#define TK_TO 45 #define TK_TO 45
#define TK_REVOKE 46 #define TK_REVOKE 46
#define TK_FROM 47 #define TK_FROM 47
#define TK_SUBSCRIBE 48 #define TK_SUBSCRIBE 48
#define TK_READ 49 #define TK_READ 49
#define TK_WRITE 50 #define TK_WRITE 50
#define TK_NK_DOT 51 #define TK_NK_DOT 51
#define TK_WITH 52 #define TK_WITH 52
#define TK_ENCRYPT_KEY 53 #define TK_ENCRYPT_KEY 53
#define TK_ANODE 54 #define TK_ANODE 54
#define TK_UPDATE 55 #define TK_UPDATE 55
#define TK_ANODES 56 #define TK_ANODES 56
#define TK_DNODE 57 #define TK_DNODE 57
#define TK_PORT 58 #define TK_PORT 58
#define TK_DNODES 59 #define TK_DNODES 59
#define TK_RESTORE 60 #define TK_RESTORE 60
#define TK_NK_IPTOKEN 61 #define TK_NK_IPTOKEN 61
#define TK_FORCE 62 #define TK_FORCE 62
#define TK_UNSAFE 63 #define TK_UNSAFE 63
#define TK_CLUSTER 64 #define TK_CLUSTER 64
#define TK_LOCAL 65 #define TK_LOCAL 65
#define TK_QNODE 66 #define TK_QNODE 66
#define TK_BNODE 67 #define TK_BNODE 67
#define TK_SNODE 68 #define TK_SNODE 68
#define TK_MNODE 69 #define TK_MNODE 69
#define TK_VNODE 70 #define TK_VNODE 70
#define TK_DATABASE 71 #define TK_DATABASE 71
#define TK_USE 72 #define TK_USE 72
#define TK_FLUSH 73 #define TK_FLUSH 73
#define TK_TRIM 74 #define TK_TRIM 74
#define TK_S3MIGRATE 75 #define TK_S3MIGRATE 75
#define TK_COMPACT 76 #define TK_COMPACT 76
#define TK_IF 77 #define TK_IF 77
#define TK_NOT 78 #define TK_NOT 78
#define TK_EXISTS 79 #define TK_EXISTS 79
#define TK_BUFFER 80 #define TK_BUFFER 80
#define TK_CACHEMODEL 81 #define TK_CACHEMODEL 81
#define TK_CACHESIZE 82 #define TK_CACHESIZE 82
#define TK_COMP 83 #define TK_COMP 83
#define TK_DURATION 84 #define TK_DURATION 84
#define TK_NK_VARIABLE 85 #define TK_NK_VARIABLE 85
#define TK_MAXROWS 86 #define TK_MAXROWS 86
#define TK_MINROWS 87 #define TK_MINROWS 87
#define TK_KEEP 88 #define TK_KEEP 88
#define TK_PAGES 89 #define TK_PAGES 89
#define TK_PAGESIZE 90 #define TK_PAGESIZE 90
#define TK_TSDB_PAGESIZE 91 #define TK_TSDB_PAGESIZE 91
#define TK_PRECISION 92 #define TK_PRECISION 92
#define TK_REPLICA 93 #define TK_REPLICA 93
#define TK_VGROUPS 94 #define TK_VGROUPS 94
#define TK_SINGLE_STABLE 95 #define TK_SINGLE_STABLE 95
#define TK_RETENTIONS 96 #define TK_RETENTIONS 96
#define TK_SCHEMALESS 97 #define TK_SCHEMALESS 97
#define TK_WAL_LEVEL 98 #define TK_WAL_LEVEL 98
#define TK_WAL_FSYNC_PERIOD 99 #define TK_WAL_FSYNC_PERIOD 99
#define TK_WAL_RETENTION_PERIOD 100 #define TK_WAL_RETENTION_PERIOD 100
#define TK_WAL_RETENTION_SIZE 101 #define TK_WAL_RETENTION_SIZE 101
#define TK_WAL_ROLL_PERIOD 102 #define TK_WAL_ROLL_PERIOD 102
#define TK_WAL_SEGMENT_SIZE 103 #define TK_WAL_SEGMENT_SIZE 103
#define TK_STT_TRIGGER 104 #define TK_STT_TRIGGER 104
#define TK_TABLE_PREFIX 105 #define TK_TABLE_PREFIX 105
#define TK_TABLE_SUFFIX 106 #define TK_TABLE_SUFFIX 106
#define TK_S3_CHUNKSIZE 107 #define TK_S3_CHUNKPAGES 107
#define TK_S3_KEEPLOCAL 108 #define TK_S3_KEEPLOCAL 108
#define TK_S3_COMPACT 109 #define TK_S3_COMPACT 109
#define TK_KEEP_TIME_OFFSET 110 #define TK_KEEP_TIME_OFFSET 110
#define TK_ENCRYPT_ALGORITHM 111 #define TK_ENCRYPT_ALGORITHM 111
#define TK_NK_COLON 112 #define TK_NK_COLON 112
#define TK_BWLIMIT 113 #define TK_BWLIMIT 113
#define TK_START 114 #define TK_START 114
#define TK_TIMESTAMP 115 #define TK_TIMESTAMP 115
#define TK_END 116 #define TK_END 116
#define TK_TABLE 117 #define TK_TABLE 117
#define TK_NK_LP 118 #define TK_NK_LP 118
#define TK_NK_RP 119 #define TK_NK_RP 119
#define TK_USING 120 #define TK_USING 120
#define TK_FILE 121 #define TK_FILE 121
#define TK_STABLE 122 #define TK_STABLE 122
#define TK_COLUMN 123 #define TK_COLUMN 123
#define TK_MODIFY 124 #define TK_MODIFY 124
#define TK_RENAME 125 #define TK_RENAME 125
#define TK_TAG 126 #define TK_TAG 126
#define TK_SET 127 #define TK_SET 127
#define TK_NK_EQ 128 #define TK_NK_EQ 128
#define TK_TAGS 129 #define TK_TAGS 129
#define TK_BOOL 130 #define TK_BOOL 130
#define TK_TINYINT 131 #define TK_TINYINT 131
#define TK_SMALLINT 132 #define TK_SMALLINT 132
#define TK_INT 133 #define TK_INT 133
#define TK_INTEGER 134 #define TK_INTEGER 134
#define TK_BIGINT 135 #define TK_BIGINT 135
#define TK_FLOAT 136 #define TK_FLOAT 136
#define TK_DOUBLE 137 #define TK_DOUBLE 137
#define TK_BINARY 138 #define TK_BINARY 138
#define TK_NCHAR 139 #define TK_NCHAR 139
#define TK_UNSIGNED 140 #define TK_UNSIGNED 140
#define TK_JSON 141 #define TK_JSON 141
#define TK_VARCHAR 142 #define TK_VARCHAR 142
#define TK_MEDIUMBLOB 143 #define TK_MEDIUMBLOB 143
#define TK_BLOB 144 #define TK_BLOB 144
#define TK_VARBINARY 145 #define TK_VARBINARY 145
#define TK_GEOMETRY 146 #define TK_GEOMETRY 146
#define TK_DECIMAL 147 #define TK_DECIMAL 147
#define TK_COMMENT 148 #define TK_COMMENT 148
#define TK_MAX_DELAY 149 #define TK_MAX_DELAY 149
#define TK_WATERMARK 150 #define TK_WATERMARK 150
#define TK_ROLLUP 151 #define TK_ROLLUP 151
#define TK_TTL 152 #define TK_TTL 152
#define TK_SMA 153 #define TK_SMA 153
#define TK_DELETE_MARK 154 #define TK_DELETE_MARK 154
#define TK_FIRST 155 #define TK_FIRST 155
#define TK_LAST 156 #define TK_LAST 156
#define TK_SHOW 157 #define TK_SHOW 157
#define TK_FULL 158 #define TK_FULL 158
#define TK_PRIVILEGES 159 #define TK_PRIVILEGES 159
#define TK_DATABASES 160 #define TK_DATABASES 160
#define TK_TABLES 161 #define TK_TABLES 161
#define TK_STABLES 162 #define TK_STABLES 162
#define TK_MNODES 163 #define TK_MNODES 163
#define TK_QNODES 164 #define TK_QNODES 164
#define TK_ARBGROUPS 165 #define TK_ARBGROUPS 165
#define TK_FUNCTIONS 166 #define TK_FUNCTIONS 166
#define TK_INDEXES 167 #define TK_INDEXES 167
#define TK_ACCOUNTS 168 #define TK_ACCOUNTS 168
#define TK_APPS 169 #define TK_APPS 169
#define TK_CONNECTIONS 170 #define TK_CONNECTIONS 170
#define TK_LICENCES 171 #define TK_LICENCES 171
#define TK_GRANTS 172 #define TK_GRANTS 172
#define TK_LOGS 173 #define TK_LOGS 173
#define TK_MACHINES 174 #define TK_MACHINES 174
#define TK_ENCRYPTIONS 175 #define TK_ENCRYPTIONS 175
#define TK_QUERIES 176 #define TK_QUERIES 176
#define TK_SCORES 177 #define TK_SCORES 177
#define TK_TOPICS 178 #define TK_TOPICS 178
#define TK_VARIABLES 179 #define TK_VARIABLES 179
#define TK_BNODES 180 #define TK_BNODES 180
#define TK_SNODES 181 #define TK_SNODES 181
#define TK_TRANSACTIONS 182 #define TK_TRANSACTIONS 182
#define TK_DISTRIBUTED 183 #define TK_DISTRIBUTED 183
#define TK_CONSUMERS 184 #define TK_CONSUMERS 184
#define TK_SUBSCRIPTIONS 185 #define TK_SUBSCRIPTIONS 185
#define TK_VNODES 186 #define TK_VNODES 186
#define TK_ALIVE 187 #define TK_ALIVE 187
#define TK_VIEWS 188 #define TK_VIEWS 188
#define TK_VIEW 189 #define TK_VIEW 189
#define TK_COMPACTS 190 #define TK_COMPACTS 190
#define TK_NORMAL 191 #define TK_NORMAL 191
#define TK_CHILD 192 #define TK_CHILD 192
#define TK_LIKE 193 #define TK_LIKE 193
#define TK_TBNAME 194 #define TK_TBNAME 194
#define TK_QTAGS 195 #define TK_QTAGS 195
#define TK_AS 196 #define TK_AS 196
#define TK_SYSTEM 197 #define TK_SYSTEM 197
#define TK_TSMA 198 #define TK_TSMA 198
#define TK_INTERVAL 199 #define TK_INTERVAL 199
#define TK_RECURSIVE 200 #define TK_RECURSIVE 200
#define TK_TSMAS 201 #define TK_TSMAS 201
#define TK_FUNCTION 202 #define TK_FUNCTION 202
#define TK_INDEX 203 #define TK_INDEX 203
#define TK_COUNT 204 #define TK_COUNT 204
#define TK_LAST_ROW 205 #define TK_LAST_ROW 205
#define TK_META 206 #define TK_META 206
#define TK_ONLY 207 #define TK_ONLY 207
#define TK_TOPIC 208 #define TK_TOPIC 208
#define TK_CONSUMER 209 #define TK_CONSUMER 209
#define TK_GROUP 210 #define TK_GROUP 210
#define TK_DESC 211 #define TK_DESC 211
#define TK_DESCRIBE 212 #define TK_DESCRIBE 212
#define TK_RESET 213 #define TK_RESET 213
#define TK_QUERY 214 #define TK_QUERY 214
#define TK_CACHE 215 #define TK_CACHE 215
#define TK_EXPLAIN 216 #define TK_EXPLAIN 216
#define TK_ANALYZE 217 #define TK_ANALYZE 217
#define TK_VERBOSE 218 #define TK_VERBOSE 218
#define TK_NK_BOOL 219 #define TK_NK_BOOL 219
#define TK_RATIO 220 #define TK_RATIO 220
#define TK_NK_FLOAT 221 #define TK_NK_FLOAT 221
#define TK_OUTPUTTYPE 222 #define TK_OUTPUTTYPE 222
#define TK_AGGREGATE 223 #define TK_AGGREGATE 223
#define TK_BUFSIZE 224 #define TK_BUFSIZE 224
#define TK_LANGUAGE 225 #define TK_LANGUAGE 225
#define TK_REPLACE 226 #define TK_REPLACE 226
#define TK_STREAM 227 #define TK_STREAM 227
#define TK_INTO 228 #define TK_INTO 228
#define TK_PAUSE 229 #define TK_PAUSE 229
#define TK_RESUME 230 #define TK_RESUME 230
#define TK_PRIMARY 231 #define TK_PRIMARY 231
#define TK_KEY 232 #define TK_KEY 232
#define TK_TRIGGER 233 #define TK_TRIGGER 233
#define TK_AT_ONCE 234 #define TK_AT_ONCE 234
#define TK_WINDOW_CLOSE 235 #define TK_WINDOW_CLOSE 235
#define TK_IGNORE 236 #define TK_IGNORE 236
#define TK_EXPIRED 237 #define TK_EXPIRED 237
#define TK_FILL_HISTORY 238 #define TK_FILL_HISTORY 238
#define TK_SUBTABLE 239 #define TK_SUBTABLE 239
#define TK_UNTREATED 240 #define TK_UNTREATED 240
#define TK_KILL 241 #define TK_KILL 241
#define TK_CONNECTION 242 #define TK_CONNECTION 242
#define TK_TRANSACTION 243 #define TK_TRANSACTION 243
#define TK_BALANCE 244 #define TK_BALANCE 244
#define TK_VGROUP 245 #define TK_VGROUP 245
#define TK_LEADER 246 #define TK_LEADER 246
#define TK_MERGE 247 #define TK_MERGE 247
#define TK_REDISTRIBUTE 248 #define TK_REDISTRIBUTE 248
#define TK_SPLIT 249 #define TK_SPLIT 249
#define TK_DELETE 250 #define TK_DELETE 250
#define TK_INSERT 251 #define TK_INSERT 251
#define TK_NK_BIN 252 #define TK_NK_BIN 252
#define TK_NK_HEX 253 #define TK_NK_HEX 253
#define TK_NULL 254 #define TK_NULL 254
#define TK_NK_QUESTION 255 #define TK_NK_QUESTION 255
#define TK_NK_ALIAS 256 #define TK_NK_ALIAS 256
#define TK_NK_ARROW 257 #define TK_NK_ARROW 257
#define TK_ROWTS 258 #define TK_ROWTS 258
#define TK_QSTART 259 #define TK_QSTART 259
#define TK_QEND 260 #define TK_QEND 260
#define TK_QDURATION 261 #define TK_QDURATION 261
#define TK_WSTART 262 #define TK_WSTART 262
#define TK_WEND 263 #define TK_WEND 263
#define TK_WDURATION 264 #define TK_WDURATION 264
#define TK_IROWTS 265 #define TK_IROWTS 265
#define TK_ISFILLED 266 #define TK_ISFILLED 266
#define TK_FLOW 267 #define TK_FLOW 267
#define TK_FHIGH 268 #define TK_FHIGH 268
#define TK_FROWTS 269 #define TK_FROWTS 269
#define TK_CAST 270 #define TK_CAST 270
#define TK_POSITION 271 #define TK_POSITION 271
#define TK_IN 272 #define TK_IN 272
#define TK_FOR 273 #define TK_FOR 273
#define TK_NOW 274 #define TK_NOW 274
#define TK_TODAY 275 #define TK_TODAY 275
#define TK_RAND 276 #define TK_RAND 276
#define TK_SUBSTR 277 #define TK_SUBSTR 277
#define TK_SUBSTRING 278 #define TK_SUBSTRING 278
#define TK_BOTH 279 #define TK_BOTH 279
#define TK_TRAILING 280 #define TK_TRAILING 280
#define TK_LEADING 281 #define TK_LEADING 281
#define TK_TIMEZONE 282 #define TK_TIMEZONE 282
#define TK_CLIENT_VERSION 283 #define TK_CLIENT_VERSION 283
#define TK_SERVER_VERSION 284 #define TK_SERVER_VERSION 284
#define TK_SERVER_STATUS 285 #define TK_SERVER_STATUS 285
#define TK_CURRENT_USER 286 #define TK_CURRENT_USER 286
#define TK_PI 287 #define TK_PI 287
#define TK_CASE 288 #define TK_CASE 288
#define TK_WHEN 289 #define TK_WHEN 289
#define TK_THEN 290 #define TK_THEN 290
#define TK_ELSE 291 #define TK_ELSE 291
#define TK_BETWEEN 292 #define TK_BETWEEN 292
#define TK_IS 293 #define TK_IS 293
#define TK_NK_LT 294 #define TK_NK_LT 294
#define TK_NK_GT 295 #define TK_NK_GT 295
#define TK_NK_LE 296 #define TK_NK_LE 296
#define TK_NK_GE 297 #define TK_NK_GE 297
#define TK_NK_NE 298 #define TK_NK_NE 298
#define TK_MATCH 299 #define TK_MATCH 299
#define TK_NMATCH 300 #define TK_NMATCH 300
#define TK_CONTAINS 301 #define TK_CONTAINS 301
#define TK_JOIN 302 #define TK_JOIN 302
#define TK_INNER 303 #define TK_INNER 303
#define TK_LEFT 304 #define TK_LEFT 304
#define TK_RIGHT 305 #define TK_RIGHT 305
#define TK_OUTER 306 #define TK_OUTER 306
#define TK_SEMI 307 #define TK_SEMI 307
#define TK_ANTI 308 #define TK_ANTI 308
#define TK_ASOF 309 #define TK_ASOF 309
#define TK_WINDOW 310 #define TK_WINDOW 310
#define TK_WINDOW_OFFSET 311 #define TK_WINDOW_OFFSET 311
#define TK_JLIMIT 312 #define TK_JLIMIT 312
#define TK_SELECT 313 #define TK_SELECT 313
#define TK_NK_HINT 314 #define TK_NK_HINT 314
#define TK_DISTINCT 315 #define TK_DISTINCT 315
#define TK_WHERE 316 #define TK_WHERE 316
#define TK_PARTITION 317 #define TK_PARTITION 317
#define TK_BY 318 #define TK_BY 318
#define TK_SESSION 319 #define TK_SESSION 319
#define TK_STATE_WINDOW 320 #define TK_STATE_WINDOW 320
#define TK_EVENT_WINDOW 321 #define TK_EVENT_WINDOW 321
#define TK_COUNT_WINDOW 322 #define TK_COUNT_WINDOW 322
#define TK_ANOMALY_WINDOW 323 #define TK_ANOMALY_WINDOW 323
#define TK_SLIDING 324 #define TK_SLIDING 324
#define TK_FILL 325 #define TK_FILL 325
#define TK_VALUE 326 #define TK_VALUE 326
#define TK_VALUE_F 327 #define TK_VALUE_F 327
#define TK_NONE 328 #define TK_NONE 328
#define TK_PREV 329 #define TK_PREV 329
#define TK_NULL_F 330 #define TK_NULL_F 330
#define TK_LINEAR 331 #define TK_LINEAR 331
#define TK_NEXT 332 #define TK_NEXT 332
#define TK_HAVING 333 #define TK_HAVING 333
#define TK_RANGE 334 #define TK_RANGE 334
#define TK_EVERY 335 #define TK_EVERY 335
#define TK_ORDER 336 #define TK_ORDER 336
#define TK_SLIMIT 337 #define TK_SLIMIT 337
#define TK_SOFFSET 338 #define TK_SOFFSET 338
#define TK_LIMIT 339 #define TK_LIMIT 339
#define TK_OFFSET 340 #define TK_OFFSET 340
#define TK_ASC 341 #define TK_ASC 341
#define TK_NULLS 342 #define TK_NULLS 342
#define TK_ABORT 343 #define TK_ABORT 343
#define TK_AFTER 344 #define TK_AFTER 344
#define TK_ATTACH 345 #define TK_ATTACH 345
#define TK_BEFORE 346 #define TK_BEFORE 346
#define TK_BEGIN 347 #define TK_BEGIN 347
#define TK_BITAND 348 #define TK_BITAND 348
#define TK_BITNOT 349 #define TK_BITNOT 349
#define TK_BITOR 350 #define TK_BITOR 350
#define TK_BLOCKS 351 #define TK_BLOCKS 351
#define TK_CHANGE 352 #define TK_CHANGE 352
#define TK_COMMA 353 #define TK_COMMA 353
#define TK_CONCAT 354 #define TK_CONCAT 354
#define TK_CONFLICT 355 #define TK_CONFLICT 355
#define TK_COPY 356 #define TK_COPY 356
#define TK_DEFERRED 357 #define TK_DEFERRED 357
#define TK_DELIMITERS 358 #define TK_DELIMITERS 358
#define TK_DETACH 359 #define TK_DETACH 359
#define TK_DIVIDE 360 #define TK_DIVIDE 360
#define TK_DOT 361 #define TK_DOT 361
#define TK_EACH 362 #define TK_EACH 362
#define TK_FAIL 363 #define TK_FAIL 363
#define TK_GLOB 364 #define TK_GLOB 364
#define TK_ID 365 #define TK_ID 365
#define TK_IMMEDIATE 366 #define TK_IMMEDIATE 366
#define TK_IMPORT 367 #define TK_IMPORT 367
#define TK_INITIALLY 368 #define TK_INITIALLY 368
#define TK_INSTEAD 369 #define TK_INSTEAD 369
#define TK_ISNULL 370 #define TK_ISNULL 370
#define TK_MODULES 371 #define TK_MODULES 371
#define TK_NK_BITNOT 372 #define TK_NK_BITNOT 372
#define TK_NK_SEMI 373 #define TK_NK_SEMI 373
#define TK_NOTNULL 374 #define TK_NOTNULL 374
#define TK_OF 375 #define TK_OF 375
#define TK_PLUS 376 #define TK_PLUS 376
#define TK_PRIVILEGE 377 #define TK_PRIVILEGE 377
#define TK_RAISE 378 #define TK_RAISE 378
#define TK_RESTRICT 379 #define TK_RESTRICT 379
#define TK_ROW 380 #define TK_ROW 380
#define TK_STAR 381 #define TK_STAR 381
#define TK_STATEMENT 382 #define TK_STATEMENT 382
#define TK_STRICT 383 #define TK_STRICT 383
#define TK_STRING 384 #define TK_STRING 384
#define TK_TIMES 385 #define TK_TIMES 385
#define TK_VALUES 386 #define TK_VALUES 386
#define TK_VARIABLE 387 #define TK_VARIABLE 387
#define TK_WAL 388 #define TK_WAL 388
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

2
include/libs/executor/executor.h
View File

@ -210,7 +210,7 @@ SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo);
const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo);
const char* qExtractTbnameFromTask(qTaskInfo_t tinfo); const char* qExtractTbnameFromTask(qTaskInfo_t tinfo);
void* qExtractReaderFromStreamScanner(void* scanner); void* qExtractReaderFromStreamScanner(void* scanner);

4
include/libs/parser/parser.h
View File

@ -176,8 +176,8 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS
STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int32_t msgBufLen); char* msgBuf, int32_t msgBufLen);
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash);
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* fields, int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);

6
include/util/tdef.h
View File

@ -450,13 +450,13 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024) #define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024)
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024) #define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024) #define TSDB_DEFAULT_S3_CHUNK_SIZE (128 * 1024)
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute #define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440) #define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440) #define TSDB_DEFAULT_S3_KEEP_LOCAL (365 * 1440)
#define TSDB_MIN_S3_COMPACT 0 #define TSDB_MIN_S3_COMPACT 0
#define TSDB_MAX_S3_COMPACT 1 #define TSDB_MAX_S3_COMPACT 1
#define TSDB_DEFAULT_S3_COMPACT 0 #define TSDB_DEFAULT_S3_COMPACT 1
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600

2
include/util/tlog.h
View File

@ -70,7 +70,7 @@ extern int32_t tdbDebugFlag;
extern int32_t sndDebugFlag; extern int32_t sndDebugFlag;
extern int32_t simDebugFlag; extern int32_t simDebugFlag;
extern int32_t tqClientDebug; extern int32_t tqClientDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc);
void taosCloseLog(); void taosCloseLog();

29
packaging/tools/make_install.bat
View File

@ -12,9 +12,18 @@ if exist C:\\TDengine\\data\\dnode\\dnodeCfg.json (
rem // stop and delete service rem // stop and delete service
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":stop_delete","","runas",1)(window.close) mshta vbscript:createobject("shell.application").shellexecute("%~s0",":stop_delete","","runas",1)(window.close)
echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
if exist %binary_dir%\\build\\bin\\taosadapter.exe (
echo This might take a few moment to accomplish deleting service taosd/taosadapter ...
)
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
echo This might take a few moment to accomplish deleting service taosd/taoskeeper ...
)
call :check_svc taosd call :check_svc taosd
call :check_svc taosadapter call :check_svc taosadapter
call :check_svc taoskeeper
set source_dir=%2 set source_dir=%2
set source_dir=%source_dir:/=\\% set source_dir=%source_dir:/=\\%
@ -46,6 +55,11 @@ if exist %binary_dir%\\test\\cfg\\taosadapter.toml (
copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul copy %binary_dir%\\test\\cfg\\taosadapter.toml %target_dir%\\cfg\\taosadapter.toml > nul
) )
) )
if exist %binary_dir%\\test\\cfg\\taoskeeper.toml (
if not exist %target_dir%\\cfg\\taoskeeper.toml (
copy %binary_dir%\\test\\cfg\\taoskeeper.toml %target_dir%\\cfg\\taoskeeper.toml > nul
)
)
copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul copy %source_dir%\\include\\client\\taos.h %target_dir%\\include > nul
copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul copy %source_dir%\\include\\util\\taoserror.h %target_dir%\\include > nul
copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul copy %source_dir%\\include\\libs\\function\\taosudf.h %target_dir%\\include > nul
@ -98,12 +112,15 @@ if %Enterprise% == TRUE (
copy %binary_dir%\\build\\bin\\*explorer.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\*explorer.exe %target_dir% > nul
) )
) )
copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul
copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul
if exist %binary_dir%\\build\\bin\\taosadapter.exe ( if exist %binary_dir%\\build\\bin\\taosadapter.exe (
copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\taosadapter.exe %target_dir% > nul
) )
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
copy %binary_dir%\\build\\bin\\taoskeeper.exe %target_dir% > nul
)
mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close) mshta vbscript:createobject("shell.application").shellexecute("%~s0",":hasAdmin","","runas",1)(window.close)
@ -116,6 +133,10 @@ if exist %binary_dir%\\build\\bin\\taosadapter.exe (
echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m echo To start/stop taosAdapter with administrator privileges: %ESC%[92msc start/stop taosadapter %ESC%[0m
) )
if exist %binary_dir%\\build\\bin\\taoskeeper.exe (
echo To start/stop taosKeeper with administrator privileges: %ESC%[92msc start/stop taoskeeper %ESC%[0m
)
goto :eof goto :eof
:hasAdmin :hasAdmin
@ -123,6 +144,7 @@ goto :eof
call :stop_delete call :stop_delete
call :check_svc taosd call :check_svc taosd
call :check_svc taosadapter call :check_svc taosadapter
call :check_svc taoskeeper
if exist c:\\windows\\sysnative ( if exist c:\\windows\\sysnative (
echo x86 echo x86
@ -141,6 +163,7 @@ if exist c:\\windows\\sysnative (
rem // create services rem // create services
sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND sc create "taosd" binPath= "C:\\TDengine\\taosd.exe --win_service" start= DEMAND
sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND sc create "taosadapter" binPath= "C:\\TDengine\\taosadapter.exe" start= DEMAND
sc create "taoskeeper" binPath= "C:\\TDengine\\taoskeeper.exe" start= DEMAND
set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment" set "env=HKLM\System\CurrentControlSet\Control\Session Manager\Environment"
for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\<Path\>"') do ( for /f "tokens=2*" %%I in ('reg query "%env%" /v Path ^| findstr /i "\<Path\>"') do (
@ -181,6 +204,8 @@ sc stop taosd
sc delete taosd sc delete taosd
sc stop taosadapter sc stop taosadapter
sc delete taosadapter sc delete taosadapter
sc stop taoskeeper
sc delete taoskeeper
exit /B 0 exit /B 0
:check_svc :check_svc

80
packaging/tools/make_install.sh
View File

@ -129,6 +129,13 @@ function kill_taosadapter() {
fi fi
} }
function kill_taoskeeper() {
pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then
${csudo}kill -9 $pid || :
fi
}
function kill_taosd() { function kill_taosd() {
pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}') pid=$(ps -ef | grep -w ${serverName} | grep -v "grep" | awk '{print $2}')
if [ -n "$pid" ]; then if [ -n "$pid" ]; then
@ -155,6 +162,7 @@ function install_bin() {
${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${clientName} || :
${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || :
${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosadapter || :
${csudo}rm -f ${bin_link_dir}/taoskeeper || :
${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/udfd || :
${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdemo || :
${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/taosdump || :
@ -169,6 +177,7 @@ function install_bin() {
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || :
@ -183,6 +192,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo > /dev/null 2>&1 || :
@ -197,6 +207,7 @@ function install_bin() {
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
[ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taoskeeper ] && ${csudo}cp -r ${binary_dir}/build/bin/taoskeeper ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || :
[ -f ${binary_dir}/build/bin/*explorer ] && ${csudo}cp -r ${binary_dir}/build/bin/*explorer ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/*explorer ] && ${csudo}cp -r ${binary_dir}/build/bin/*explorer ${install_main_dir}/bin || :
@ -208,6 +219,7 @@ function install_bin() {
[ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/taosadapter ] && ${csudo}ln -s ${install_main_dir}/bin/taosadapter ${bin_link_dir}/taosadapter > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taoskeeper ] && ${csudo}ln -s ${install_main_dir}/bin/taoskeeper ${bin_link_dir}/taoskeeper > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd > /dev/null 2>&1 || :
[ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump > /dev/null 2>&1 || :
[ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || : [ -f ${install_main_dir}/bin/taosBenchmark ] && ${csudo}ln -sf ${install_main_dir}/bin/taosBenchmark ${install_main_dir}/bin/taosdemo > /dev/null 2>&1 || :
@ -407,6 +419,29 @@ function install_taosadapter_config() {
fi fi
} }
function install_taoskeeper_config() {
if [ ! -f "${cfg_install_dir}/taoskeeper.toml" ]; then
${csudo}mkdir -p ${cfg_install_dir} || :
[ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_install_dir} &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml || :
[ -f ${binary_dir}/test/cfg/taoskeeper.toml ] &&
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
[ -f ${cfg_install_dir}/taoskeeper.toml ] &&
${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml \
${install_main_dir}/cfg/taoskeeper.toml > /dev/null 2>&1 || :
else
if [ -f "${binary_dir}/test/cfg/taoskeeper.toml" ]; then
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml \
${cfg_install_dir}/taoskeeper.toml.${verNumber} || :
${csudo}cp -f ${binary_dir}/test/cfg/taoskeeper.toml ${cfg_dir} || :
fi
fi
}
function install_log() { function install_log() {
${csudo}rm -rf ${log_dir} || : ${csudo}rm -rf ${log_dir} || :
${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir}
@ -526,6 +561,15 @@ function install_taosadapter_service() {
fi fi
} }
function install_taoskeeper_service() {
if ((${service_mod} == 0)); then
[ -f ${binary_dir}/test/cfg/taoskeeper.service ] &&
${csudo}cp ${binary_dir}/test/cfg/taoskeeper.service \
${service_config_dir}/ || :
${csudo}systemctl daemon-reload
fi
}
function install_service_on_launchctl() { function install_service_on_launchctl() {
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || : ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosd.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist ${csudo}cp ${script_dir}/com.taosdata.taosd.plist /Library/LaunchDaemons/com.taosdata.taosd.plist
@ -534,6 +578,10 @@ function install_service_on_launchctl() {
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : ${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist ${csudo}cp ${script_dir}/com.taosdata.taosadapter.plist /Library/LaunchDaemons/com.taosdata.taosadapter.plist
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || : ${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taosadapter.plist > /dev/null 2>&1 || :
${csudo}launchctl unload -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
${csudo}cp ${script_dir}/com.taosdata.taoskeeper.plist /Library/LaunchDaemons/com.taosdata.taoskeeper.plist
${csudo}launchctl load -w /Library/LaunchDaemons/com.taosdata.taoskeeper.plist > /dev/null 2>&1 || :
} }
function install_service() { function install_service() {
@ -549,6 +597,7 @@ function install_service() {
install_service_on_launchctl install_service_on_launchctl
fi fi
} }
function install_app() { function install_app() {
if [ "$osType" = "Darwin" ]; then if [ "$osType" = "Darwin" ]; then
${csudo}rm -rf /Applications/TDengine.app && ${csudo}rm -rf /Applications/TDengine.app &&
@ -573,6 +622,7 @@ function update_TDengine() {
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
${csudo}service ${serverName} stop || : ${csudo}service ${serverName} stop || :
else else
kill_taoskeeper
kill_taosadapter kill_taosadapter
kill_taosd kill_taosd
fi fi
@ -591,9 +641,11 @@ function update_TDengine() {
install_service install_service
install_taosadapter_service install_taosadapter_service
install_taoskeeper_service
install_config install_config
install_taosadapter_config install_taosadapter_config
install_taoskeeper_config
echo echo
echo -e "\033[44;32;1m${productName} is updated successfully!${NC}" echo -e "\033[44;32;1m${productName} is updated successfully!${NC}"
@ -602,22 +654,33 @@ function update_TDengine() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
[ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
else else
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
else else
echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
fi fi
fi fi
@ -643,9 +706,11 @@ function install_TDengine() {
install_service install_service
install_taosadapter_service install_taosadapter_service
install_taoskeeper_service
install_config install_config
install_taosadapter_config install_taosadapter_config
install_taoskeeper_config
# Ask if to start the service # Ask if to start the service
echo echo
@ -654,22 +719,33 @@ function install_TDengine() {
echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}"
[ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml"
[ -f ${configDir}/taoskeeper.toml ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To configure Keeper ${NC}: edit ${configDir}/taoskeeper.toml"
if ((${service_mod} == 0)); then if ((${service_mod} == 0)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}systemctl start taoskeeper ${NC}"
elif ((${service_mod} == 1)); then elif ((${service_mod} == 1)); then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}"
[ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}"
[ -f ${service_config_dir}/taoskeeper.service ] && [ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: ${csudo}service taoskeeper start${NC}"
else else
if [ "$osType" != "Darwin" ]; then if [ "$osType" != "Darwin" ]; then
echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}"
[ -f ${installDir}/bin/taosadapter ] && \ [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start Adapter ${NC}: taosadapter &${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: taoskeeper &${NC}"
else else
echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}" echo -e "${GREEN_DARK}To start service ${NC}: sudo launchctl start com.tdengine.taosd${NC}"
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}" [ -f ${installDir}/bin/taosadapter ] && \
echo -e "${GREEN_DARK}To start Adapter ${NC}: sudo launchctl start com.tdengine.taosadapter${NC}"
[ -f ${installDir}/bin/taoskeeper ] && \
echo -e "${GREEN_DARK}To start Keeper ${NC}: sudo launchctl start com.tdengine.taoskeeper${NC}"
fi fi
fi fi

677
source/client/src/clientRawBlockWrite.c
View File

@ -52,10 +52,8 @@
#define TMQ_META_VERSION "1.0" #define TMQ_META_VERSION "1.0"
static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen); static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen);
static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); }
static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, static void buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t,
SColCmprWrapper* pColCmprRow, cJSON** pJson) { SColCmprWrapper* pColCmprRow, cJSON** pJson) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -457,7 +455,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) {
cJSON* tvalue = NULL; cJSON* tvalue = NULL;
if (IS_VAR_DATA_TYPE(pTagVal->type)) { if (IS_VAR_DATA_TYPE(pTagVal->type)) {
char* buf = NULL; char* buf = NULL;
int64_t bufSize = 0; int64_t bufSize = 0;
if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) { if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) {
bufSize = pTagVal->nData * 2 + 2 + 3; bufSize = pTagVal->nData * 2 + 2 + 3;
@ -890,9 +888,6 @@ end:
} }
static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVCreateStbReq req = {0}; SVCreateStbReq req = {0};
SDecoder coder; SDecoder coder;
SMCreateStbReq pReq = {0}; SMCreateStbReq pReq = {0};
@ -1003,9 +998,6 @@ end:
} }
static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVDropStbReq req = {0}; SVDropStbReq req = {0};
SDecoder coder = {0}; SDecoder coder = {0};
SMDropStbReq pReq = {0}; SMDropStbReq pReq = {0};
@ -1115,9 +1107,6 @@ static void destroyCreateTbReqBatch(void* data) {
} }
static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVCreateTbBatchReq req = {0}; SVCreateTbBatchReq req = {0};
SDecoder coder = {0}; SDecoder coder = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1304,9 +1293,6 @@ static void destroyDropTbReqBatch(void* data) {
} }
static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVDropTbBatchReq req = {0}; SVDropTbBatchReq req = {0};
SDecoder coder = {0}; SDecoder coder = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1419,9 +1405,6 @@ end:
} }
static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SDeleteRes req = {0}; SDeleteRes req = {0};
SDecoder coder = {0}; SDecoder coder = {0};
char sql[256] = {0}; char sql[256] = {0};
@ -1457,9 +1440,6 @@ end:
} }
static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taos == NULL || meta == NULL) {
return TSDB_CODE_INVALID_PARA;
}
SVAlterTbReq req = {0}; SVAlterTbReq req = {0};
SDecoder dcoder = {0}; SDecoder dcoder = {0};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1622,7 +1602,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
RAW_NULL_CHECK(pVgHash); RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK( RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0)); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false, NULL, 0, false));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL); launchQueryImpl(pRequest, pQuery, true, NULL);
@ -1682,7 +1662,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
RAW_NULL_CHECK(pVgHash); RAW_NULL_CHECK(pVgHash);
RAW_RETURN_CHECK( RAW_RETURN_CHECK(
taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData))); taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)));
RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0)); RAW_RETURN_CHECK(rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false, NULL, 0, false));
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL); launchQueryImpl(pRequest, pQuery, true, NULL);
@ -1708,116 +1688,6 @@ static void* getRawDataFromRes(void* pRetrieve) {
return rawData; return rawData;
} }
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
if (taos == NULL || data == NULL) {
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data);
return TSDB_CODE_INVALID_PARA;
}
int32_t code = TSDB_CODE_SUCCESS;
SHashObj* pVgHash = NULL;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
STableMeta* pTableMeta = NULL;
SRequestObj* pRequest = NULL;
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest));
uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
pRequest->syncQuery = true;
rspObj.resIter = -1;
rspObj.resType = RES_TYPE__TMQ;
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
tDecoderInit(&decoder, data, dataLen);
code = tDecodeMqDataRsp(&decoder, &rspObj.dataRsp);
if (code != 0) {
SET_ERROR_MSG("decode mq data rsp failed");
code = TSDB_CODE_INVALID_MSG;
goto end;
}
if (!pRequest->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
goto end;
}
struct SCatalog* pCatalog = NULL;
RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog));
SRequestConnInfo conn = {0};
conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter;
conn.requestId = pRequest->requestId;
conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
RAW_NULL_CHECK(tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta));
SVgroupInfo vg = {0};
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg));
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) {
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
RAW_NULL_CHECK(fields);
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN);
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
}
}
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
end:
uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteMqDataRsp(&rspObj.dataRsp);
tDecoderClear(&decoder);
qDestroyQuery(pQuery);
destroyRequest(pRequest);
taosHashCleanup(pVgHash);
taosMemoryFreeClear(pTableMeta);
return code;
}
static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) { static int32_t buildCreateTbMap(SMqDataRsp* rsp, SHashObj* pHashObj) {
// find schema data info // find schema data info
int32_t code = 0; int32_t code = 0;
@ -1855,152 +1725,368 @@ end:
return code; return code;
} }
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { typedef enum {
if (taos == NULL || data == NULL) { WRITE_RAW_INIT_START = 0,
SET_ERROR_MSG("taos:%p or data:%p is NULL", taos, data); WRITE_RAW_INIT_OK,
return TSDB_CODE_INVALID_PARA; WRITE_RAW_INIT_FAIL,
} WRITE_RAW_INIT_STATUS;
static SHashObj* writeRawCache = NULL;
static int8_t initFlag = 0;
static int8_t initedFlag = WRITE_RAW_INIT_START;
typedef struct {
SHashObj* pVgHash;
SHashObj* pNameHash;
SHashObj* pMetaHash;
} rawCacheInfo;
typedef struct {
SVgroupInfo vgInfo;
int64_t uid;
int64_t suid;
} tbInfo;
static void tmqFreeMeta(void* data) {
STableMeta* pTableMeta = *(STableMeta**)data;
taosMemoryFree(pTableMeta);
}
static void freeRawCache(void* data) {
rawCacheInfo* pRawCache = (rawCacheInfo*)data;
taosHashCleanup(pRawCache->pMetaHash);
taosHashCleanup(pRawCache->pNameHash);
taosHashCleanup(pRawCache->pVgHash);
}
static int32_t initRawCacheHash() {
if (writeRawCache == NULL) {
writeRawCache = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (writeRawCache == NULL) {
return terrno;
}
taosHashSetFreeFp(writeRawCache, freeRawCache);
} }
int32_t code = TSDB_CODE_SUCCESS; return 0;
SHashObj* pVgHash = NULL; }
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
STableMeta* pTableMeta = NULL;
SHashObj* pCreateTbHash = NULL;
SRequestObj* pRequest = NULL; static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) {
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, &pRequest)); char* p = (char*)rawData;
// | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each
// column length |
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(int32_t);
p += sizeof(uint64_t);
int8_t* fields = p;
uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) {
pRequest->syncQuery = true; return true;
rspObj.resIter = -1; }
rspObj.resType = RES_TYPE__TMQ_METADATA; for (int i = 0; i < pSW->nCols; i++) {
int j = 0;
for (; j < pTableMeta->tableInfo.numOfColumns; j++) {
SSchema* pColSchema = &pTableMeta->schema[j];
char* fieldName = pSW->pSchema[i].name;
int8_t dataVersion = *(int8_t*)data; if (strcmp(pColSchema->name, fieldName) == 0) {
if (dataVersion >= MQ_DATA_RSP_VERSION) { if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t)); return true;
dataLen -= sizeof(int8_t) + sizeof(int32_t); }
break;
}
}
fields += sizeof(int8_t) + sizeof(int32_t);
if (j == pTableMeta->tableInfo.numOfColumns) return true;
}
return false;
}
static int32_t getRawCache(SHashObj** pVgHash, SHashObj** pNameHash, SHashObj** pMetaHash, void* key) {
int32_t code = 0;
void* cacheInfo = taosHashGet(writeRawCache, &key, POINTER_BYTES);
if (cacheInfo == NULL) {
*pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pVgHash);
*pNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pNameHash);
*pMetaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(*pMetaHash);
taosHashSetFreeFp(*pMetaHash, tmqFreeMeta);
rawCacheInfo info = {*pVgHash, *pNameHash, *pMetaHash};
RAW_RETURN_CHECK(taosHashPut(writeRawCache, &key, POINTER_BYTES, &info, sizeof(rawCacheInfo)));
} else {
rawCacheInfo* info = (rawCacheInfo*)cacheInfo;
*pVgHash = info->pVgHash;
*pNameHash = info->pNameHash;
*pMetaHash = info->pMetaHash;
} }
tDecoderInit(&decoder, data, dataLen); return 0;
code = tDecodeSTaosxRsp(&decoder, &rspObj.dataRsp); end:
if (code != 0) { taosHashCleanup(*pMetaHash);
SET_ERROR_MSG("decode mq taosx data rsp failed"); taosHashCleanup(*pNameHash);
code = TSDB_CODE_INVALID_MSG; taosHashCleanup(*pVgHash);
goto end; return code;
} }
if (!pRequest->pDb) { static int32_t buildRawRequest(TAOS* taos, SRequestObj** pRequest, SCatalog** pCatalog, SRequestConnInfo* conn) {
int32_t code = 0;
RAW_RETURN_CHECK(createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0, pRequest));
(*pRequest)->syncQuery = true;
if (!(*pRequest)->pDb) {
code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; code = TSDB_CODE_PAR_DB_NOT_SPECIFIED;
goto end; goto end;
} }
struct SCatalog* pCatalog = NULL; RAW_RETURN_CHECK(catalogGetHandle((*pRequest)->pTscObj->pAppInfo->clusterId, pCatalog));
RAW_RETURN_CHECK(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog)); conn->pTrans = (*pRequest)->pTscObj->pAppInfo->pTransporter;
conn->requestId = (*pRequest)->requestId;
conn->requestObjRefId = (*pRequest)->self;
conn->mgmtEps = getEpSet_s(&(*pRequest)->pTscObj->pAppInfo->mgmtEp);
end:
return code;
}
typedef int32_t _raw_decode_func_(SDecoder* pDecoder, SMqDataRsp* pRsp);
static int32_t decodeRawData(SDecoder* decoder, void* data, int32_t dataLen, _raw_decode_func_ func,
SMqRspObj* rspObj) {
int8_t dataVersion = *(int8_t*)data;
if (dataVersion >= MQ_DATA_RSP_VERSION) {
data = POINTER_SHIFT(data, sizeof(int8_t) + sizeof(int32_t));
dataLen -= sizeof(int8_t) + sizeof(int32_t);
}
rspObj->resIter = -1;
tDecoderInit(decoder, data, dataLen);
int32_t code = func(decoder, &rspObj->dataRsp);
if (code != 0) {
SET_ERROR_MSG("decode mq taosx data rsp failed");
}
return code;
}
static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj* pMetaHash,
SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName,
STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) {
int32_t code = 0;
STableMeta* pTableMeta = NULL;
tbInfo* tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
if (tmpInfo == NULL || retry > 0) {
tbInfo info = {0};
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, conn, pName, &info.vgInfo));
if (pCreateReqDst && tmpInfo == NULL) { // change stable name to get meta
tstrncpy(pName->tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN);
}
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
info.uid = pTableMeta->uid;
if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
info.suid = pTableMeta->suid;
} else {
info.suid = pTableMeta->uid;
}
code = taosHashPut(pMetaHash, &info.suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
if (code != 0) {
taosMemoryFree(pTableMeta);
goto end;
}
if (pCreateReqDst) {
pTableMeta->vgId = info.vgInfo.vgId;
pTableMeta->uid = pCreateReqDst->uid;
pCreateReqDst->ctb.suid = pTableMeta->suid;
}
RAW_RETURN_CHECK(taosHashPut(pNameHash, pName->tname, strlen(pName->tname), &info, sizeof(tbInfo)));
tmpInfo = (tbInfo*)taosHashGet(pNameHash, pName->tname, strlen(pName->tname));
RAW_RETURN_CHECK(
taosHashPut(pVgHash, &info.vgInfo.vgId, sizeof(info.vgInfo.vgId), &info.vgInfo, sizeof(SVgroupInfo)));
}
if (pTableMeta == NULL || retry > 0) {
STableMeta** pTableMetaTmp = (STableMeta**)taosHashGet(pMetaHash, &tmpInfo->suid, LONG_BYTES);
if (pTableMetaTmp == NULL || retry > 0 || needRefreshMeta(rawData, *pTableMetaTmp, pSW)) {
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, conn, pName, &pTableMeta));
code = taosHashPut(pMetaHash, &tmpInfo->suid, LONG_BYTES, &pTableMeta, POINTER_BYTES);
if (code != 0) {
taosMemoryFree(pTableMeta);
goto end;
}
} else {
pTableMeta = *pTableMetaTmp;
pTableMeta->uid = tmpInfo->uid;
pTableMeta->vgId = tmpInfo->vgInfo.vgId;
}
}
*pMeta = pTableMeta;
end:
return code;
}
static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
int32_t code = TSDB_CODE_SUCCESS;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
SRequestObj* pRequest = NULL;
SCatalog* pCatalog = NULL;
SRequestConnInfo conn = {0}; SRequestConnInfo conn = {0};
conn.pTrans = pRequest->pTscObj->pAppInfo->pTransporter; RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
conn.requestId = pRequest->requestId; uDebug(LOG_ID_TAG " write raw data, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
conn.requestObjRefId = pRequest->self; RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj));
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
SHashObj* pVgHash = NULL;
SHashObj* pNameHash = NULL;
SHashObj* pMetaHash = NULL;
RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
int retry = 0;
while (1) {
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
RAW_NULL_CHECK(tbName);
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
RAW_NULL_CHECK(pRetrieve);
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
STableMeta* pTableMeta = NULL;
RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, &pTableMeta, pSW,
rawData, retry));
char err[ERR_MSG_LEN] = {0};
code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
goto end;
}
}
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
qDestroyQuery(pQuery);
pQuery = NULL;
rspObj.resIter = -1;
continue;
}
break;
}
end:
uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteMqDataRsp(&rspObj.dataRsp);
tDecoderClear(&decoder);
qDestroyQuery(pQuery);
destroyRequest(pRequest);
return code;
}
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
int32_t code = TSDB_CODE_SUCCESS;
SQuery* pQuery = NULL;
SMqRspObj rspObj = {0};
SDecoder decoder = {0};
SHashObj* pCreateTbHash = NULL;
SRequestObj* pRequest = NULL;
SCatalog* pCatalog = NULL;
SRequestConnInfo conn = {0};
RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn));
uDebug(LOG_ID_TAG " write raw metadata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen);
RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeSTaosxRsp, &rspObj));
RAW_RETURN_CHECK(smlInitHandle(&pQuery));
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pVgHash);
pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pCreateTbHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
RAW_NULL_CHECK(pCreateTbHash); RAW_NULL_CHECK(pCreateTbHash);
RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash)); RAW_RETURN_CHECK(buildCreateTbMap(&rspObj.dataRsp, pCreateTbHash));
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum); SHashObj* pVgHash = NULL;
while (++rspObj.resIter < rspObj.dataRsp.blockNum) { SHashObj* pNameHash = NULL;
void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); SHashObj* pMetaHash = NULL;
RAW_NULL_CHECK(pRetrieve); RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos));
if (!rspObj.dataRsp.withSchema) { int retry = 0;
goto end; while (1) {
} RAW_RETURN_CHECK(smlInitHandle(&pQuery));
uDebug(LOG_ID_TAG " write raw meta data block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum);
while (++rspObj.resIter < rspObj.dataRsp.blockNum) {
if (!rspObj.dataRsp.withSchema) {
goto end;
}
const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter);
if (!tbName) { RAW_NULL_CHECK(tbName);
SET_ERROR_MSG("block tbname is null"); SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
code = terrno; RAW_NULL_CHECK(pSW);
goto end; void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter);
} RAW_NULL_CHECK(pRetrieve);
void* rawData = getRawDataFromRes(pRetrieve);
RAW_NULL_CHECK(rawData);
uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName);
SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}};
tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN);
tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN);
// find schema data info // find schema data info
SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, tbName, strlen(tbName)); SVCreateTbReq* pCreateReqDst = (SVCreateTbReq*)taosHashGet(pCreateTbHash, pName.tname, strlen(pName.tname));
SVgroupInfo vg = {0}; STableMeta* pTableMeta = NULL;
RAW_RETURN_CHECK(catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg)); RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, pCreateReqDst, pCatalog, &conn, &pName,
if (pCreateReqDst) { // change stable name to get meta &pTableMeta, pSW, rawData, retry));
tstrncpy(pName.tname, pCreateReqDst->ctb.stbName, TSDB_TABLE_NAME_LEN); char err[ERR_MSG_LEN] = {0};
code =
rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, pSW, pSW->nCols, true, err, ERR_MSG_LEN, true);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", pName.tname, err);
goto end;
}
} }
RAW_RETURN_CHECK(catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta)); RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
if (pCreateReqDst) { if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) {
pTableMeta->vgId = vg.vgId; uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code));
pTableMeta->uid = pCreateReqDst->uid; qDestroyQuery(pQuery);
pCreateReqDst->ctb.suid = pTableMeta->suid; pQuery = NULL;
} rspObj.resIter = -1;
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); continue;
if (hData == NULL) {
RAW_RETURN_CHECK(taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
}
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.dataRsp.blockSchema, rspObj.resIter);
RAW_NULL_CHECK(pSW);
TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD));
if (fields == NULL) {
SET_ERROR_MSG("calloc fields failed");
code = terrno;
goto end;
}
for (int i = 0; i < pSW->nCols; i++) {
fields[i].type = pSW->pSchema[i].type;
fields[i].bytes = pSW->pSchema[i].bytes;
tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name));
}
void* rawData = getRawDataFromRes(pRetrieve);
char err[ERR_MSG_LEN] = {0};
SVCreateTbReq* pCreateReqTmp = NULL;
if (pCreateReqDst) {
RAW_RETURN_CHECK(cloneSVreateTbReq(pCreateReqDst, &pCreateReqTmp));
}
code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqTmp, fields, pSW->nCols, true, err, ERR_MSG_LEN);
if (pCreateReqTmp != NULL) {
tdDestroySVCreateTbReq(pCreateReqTmp);
taosMemoryFree(pCreateReqTmp);
}
taosMemoryFree(fields);
taosMemoryFreeClear(pTableMeta);
if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("table:%s, err:%s", tbName, err);
goto end;
} }
break;
} }
RAW_RETURN_CHECK(smlBuildOutput(pQuery, pVgHash));
launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code;
end: end:
uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code));
tDeleteSTaosxRsp(&rspObj.dataRsp);
void* pIter = taosHashIterate(pCreateTbHash, NULL); void* pIter = taosHashIterate(pCreateTbHash, NULL);
while (pIter) { while (pIter) {
tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE); tDestroySVCreateTbReq(pIter, TSDB_MSG_FLG_DECODE);
pIter = taosHashIterate(pCreateTbHash, pIter); pIter = taosHashIterate(pCreateTbHash, pIter);
} }
taosHashCleanup(pCreateTbHash); taosHashCleanup(pCreateTbHash);
tDeleteSTaosxRsp(&rspObj.dataRsp);
tDecoderClear(&decoder); tDecoderClear(&decoder);
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
destroyRequest(pRequest); destroyRequest(pRequest);
taosHashCleanup(pVgHash);
taosMemoryFreeClear(pTableMeta);
return code; return code;
} }
@ -2076,18 +2162,18 @@ char* tmq_get_json_meta(TAOS_RES* res) {
return NULL; return NULL;
} }
char* string = NULL; char* string = NULL;
SMqRspObj* rspObj = (SMqRspObj*)res; SMqRspObj* rspObj = (SMqRspObj*)res;
if (TD_RES_TMQ_METADATA(res)) { if (TD_RES_TMQ_METADATA(res)) {
processAutoCreateTable(&rspObj->dataRsp, &string); processAutoCreateTable(&rspObj->dataRsp, &string);
} else if (TD_RES_TMQ_BATCH_META(res)) { } else if (TD_RES_TMQ_BATCH_META(res)) {
processBatchMetaToJson(&rspObj->batchMetaRsp, &string); processBatchMetaToJson(&rspObj->batchMetaRsp, &string);
} else if (TD_RES_TMQ_META(res)) { } else if (TD_RES_TMQ_META(res)) {
cJSON* pJson = NULL; cJSON* pJson = NULL;
processSimpleMeta(&rspObj->metaRsp, &pJson); processSimpleMeta(&rspObj->metaRsp, &pJson);
string = cJSON_PrintUnformatted(pJson); string = cJSON_PrintUnformatted(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
} else{ } else {
uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res); uError("tmq_get_json_meta res:%d, invalid type", *(int8_t*)res);
} }
@ -2098,7 +2184,7 @@ char* tmq_get_json_meta(TAOS_RES* res) {
void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); }
static int32_t getOffSetLen(const SMqDataRsp* pRsp) { static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
SEncoder coder = {0}; SEncoder coder = {0};
tEncoderInit(&coder, NULL, 0); tEncoderInit(&coder, NULL, 0);
if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(&coder, &pRsp->reqOffset) < 0) return -1;
if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1; if (tEncodeSTqOffsetVal(&coder, &pRsp->rspOffset) < 0) return -1;
@ -2108,45 +2194,45 @@ static int32_t getOffSetLen(const SMqDataRsp* pRsp) {
} }
typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp); typedef int32_t __encode_func__(SEncoder* pEncoder, const SMqDataRsp* pRsp);
static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) { static int32_t encodeMqDataRsp(__encode_func__* encodeFunc, SMqDataRsp* rspObj, tmq_raw_data* raw) {
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
SEncoder encoder = {0}; SEncoder encoder = {0};
void* buf = NULL; void* buf = NULL;
tEncodeSize(encodeFunc, rspObj, len, code); tEncodeSize(encodeFunc, rspObj, len, code);
if (code < 0) { if (code < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto FAILED; goto FAILED;
} }
len += sizeof(int8_t) + sizeof(int32_t); len += sizeof(int8_t) + sizeof(int32_t);
buf = taosMemoryCalloc(1, len); buf = taosMemoryCalloc(1, len);
if (buf == NULL) { if (buf == NULL) {
code = terrno; code = terrno;
goto FAILED; goto FAILED;
} }
tEncoderInit(&encoder, buf, len); tEncoderInit(&encoder, buf, len);
if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) { if (tEncodeI8(&encoder, MQ_DATA_RSP_VERSION) < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto FAILED; goto FAILED;
} }
int32_t offsetLen = getOffSetLen(rspObj); int32_t offsetLen = getOffSetLen(rspObj);
if (offsetLen <= 0) { if (offsetLen <= 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto FAILED; goto FAILED;
} }
if (tEncodeI32(&encoder, offsetLen) < 0) { if (tEncodeI32(&encoder, offsetLen) < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto FAILED; goto FAILED;
} }
if (encodeFunc(&encoder, rspObj) < 0) { if (encodeFunc(&encoder, rspObj) < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto FAILED; goto FAILED;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
raw->raw = buf; raw->raw = buf;
raw->raw_len = len; raw->raw_len = len;
return code; return code;
FAILED: FAILED:
tEncoderClear(&encoder); tEncoderClear(&encoder);
taosMemoryFree(buf); taosMemoryFree(buf);
@ -2164,7 +2250,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) {
raw->raw_type = rspObj->metaRsp.resMsgType; raw->raw_type = rspObj->metaRsp.resMsgType;
uDebug("tmq get raw type meta:%p", raw); uDebug("tmq get raw type meta:%p", raw);
} else if (TD_RES_TMQ(res)) { } else if (TD_RES_TMQ(res)) {
int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw); int32_t code = encodeMqDataRsp(tEncodeMqDataRsp, &rspObj->dataRsp, raw);
if (code != 0) { if (code != 0) {
uError("tmq get raw type error:%d", terrno); uError("tmq get raw type error:%d", terrno);
return code; return code;
@ -2199,7 +2285,31 @@ void tmq_free_raw(tmq_raw_data raw) {
(void)memset(terrMsg, 0, ERR_MSG_LEN); (void)memset(terrMsg, 0, ERR_MSG_LEN);
} }
static int32_t writeRawInit() {
while (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_START) {
int8_t old = atomic_val_compare_exchange_8(&initFlag, 0, 1);
if (old == 0) {
int32_t code = initRawCacheHash();
if (code != 0) {
uError("tmq writeRawImpl init error:%d", code);
atomic_store_8(&initedFlag, WRITE_RAW_INIT_FAIL);
return code;
}
atomic_store_8(&initedFlag, WRITE_RAW_INIT_OK);
}
}
if (atomic_load_8(&initedFlag) == WRITE_RAW_INIT_FAIL) {
return TSDB_CODE_INTERNAL_ERROR;
}
return 0;
}
static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) { static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) {
if (writeRawInit() != 0) {
return TSDB_CODE_INTERNAL_ERROR;
}
if (type == TDMT_VND_CREATE_STB) { if (type == TDMT_VND_CREATE_STB) {
return taosCreateStb(taos, buf, len); return taosCreateStb(taos, buf, len);
} else if (type == TDMT_VND_ALTER_STB) { } else if (type == TDMT_VND_ALTER_STB) {
@ -2214,10 +2324,10 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
return taosDropTable(taos, buf, len); return taosDropTable(taos, buf, len);
} else if (type == TDMT_VND_DELETE) { } else if (type == TDMT_VND_DELETE) {
return taosDeleteData(taos, buf, len); return taosDeleteData(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_METADATA) { } else if (type == RES_TYPE__TMQ_METADATA) {
return tmqWriteRawMetaDataImpl(taos, buf, len); return tmqWriteRawMetaDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ) {
return tmqWriteRawDataImpl(taos, buf, len);
} else if (type == RES_TYPE__TMQ_BATCH_META) { } else if (type == RES_TYPE__TMQ_BATCH_META) {
return tmqWriteBatchMetaDataImpl(taos, buf, len); return tmqWriteBatchMetaDataImpl(taos, buf, len);
} }
@ -2225,7 +2335,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type)
} }
int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) {
if (!taos) { if (taos == NULL || raw.raw == NULL || raw.raw_len <= 0) {
SET_ERROR_MSG("taos:%p or data:%p is NULL or raw_len <= 0", taos, raw.raw);
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }

29
source/client/src/clientTmq.c
View File

@ -24,12 +24,9 @@
#include "tref.h" #include "tref.h"
#include "ttimer.h" #include "ttimer.h"
#define tqFatalC(...) do { if (cDebugFlag & DEBUG_FATAL || tqClientDebug) { taosPrintLog("TQ FATAL ", DEBUG_FATAL, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define tqErrorC(...) do { if (cDebugFlag & DEBUG_ERROR || tqClientDebug) { taosPrintLog("TQ ERROR ", DEBUG_ERROR, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define tqWarnC(...) do { if (cDebugFlag & DEBUG_WARN || tqClientDebug) { taosPrintLog("TQ WARN ", DEBUG_WARN, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqClientDebugFlag|cDebugFlag, __VA_ARGS__); }} while(0)
#define tqInfoC(...) do { if (cDebugFlag & DEBUG_INFO || tqClientDebug) { taosPrintLog("TQ ", DEBUG_INFO, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqDebugC(...) do { if (cDebugFlag & DEBUG_DEBUG || tqClientDebug) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0)
#define tqTraceC(...) do { if (cDebugFlag & DEBUG_TRACE || tqClientDebug) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0)
#define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define EMPTY_BLOCK_POLL_IDLE_DURATION 10
#define DEFAULT_AUTO_COMMIT_INTERVAL 5000 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000
@ -831,8 +828,8 @@ static int32_t innerCommitAll(tmq_t* tmq, SMqCommitCbParamSet* pParamSet){
} }
code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet); code = innerCommit(tmq, pTopic->topicName, &pVg->offsetInfo.endOffset, pVg, pParamSet);
if (code != 0){ if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
tqDebugC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d", tqErrorC("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, code:%s, current offset version:%" PRId64 ", ordinal:%d/%d",
tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); tmq->consumerId, pTopic->topicName, pVg->vgId, tstrerror(code), pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups);
} }
} }
@ -857,7 +854,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us
return; return;
} }
code = innerCommitAll(tmq, pParamSet); code = innerCommitAll(tmq, pParamSet);
if (code != 0){ if (code != 0 && code != TSDB_CODE_TMQ_SAME_COMMITTED_VALUE){
tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code)); tqErrorC("consumer:0x%" PRIx64 " innerCommitAll failed, code:%s", tmq->consumerId, tstrerror(code));
} }
@ -957,7 +954,8 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
} }
} }
tqClientDebug = rsp.debugFlag; tqClientDebugFlag = rsp.debugFlag;
tDestroySMqHbRsp(&rsp); tDestroySMqHbRsp(&rsp);
END: END:
@ -978,6 +976,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
req.consumerId = tmq->consumerId; req.consumerId = tmq->consumerId;
req.epoch = tmq->epoch; req.epoch = tmq->epoch;
req.pollFlag = atomic_load_8(&tmq->pollFlag); req.pollFlag = atomic_load_8(&tmq->pollFlag);
tqDebugC("consumer:0x%" PRIx64 " send heartbeat, pollFlag:%d", tmq->consumerId, req.pollFlag);
req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows));
if (req.topics == NULL) { if (req.topics == NULL) {
goto END; goto END;
@ -1063,7 +1062,7 @@ END:
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
if (tmrId != NULL) { if (tmrId != NULL) {
bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer); bool ret = taosTmrReset(tmqSendHbReq, tmq->heartBeatIntervalMs, param, tmqMgmt.timer, &tmq->hbLiveTimer);
tqDebugC("reset timer fo tmq hb:%d", ret); tqDebugC("consumer:0x%" PRIx64 " reset timer for tmq heartbeat:%d, pollFlag:%d", tmq->consumerId, ret, tmq->pollFlag);
} }
int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId); int32_t ret = taosReleaseRef(tmqMgmt.rsetId, refId);
if (ret != 0){ if (ret != 0){
@ -1269,7 +1268,9 @@ static int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code)); if (code != TSDB_CODE_MND_CONSUMER_NOT_READY){
tqErrorC("consumer:0x%" PRIx64 ", get topic endpoint error, code:%s", tmq->consumerId, tstrerror(code));
}
goto END; goto END;
} }
@ -1422,7 +1423,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); tqDebugC("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId);
bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer, bool ret = taosTmrReset(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->epTimer); &pTmq->epTimer);
tqDebugC("reset timer fo tmq ask ep:%d", ret); tqDebugC("reset timer for tmq ask ep:%d", ret);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn; tmq_commit_cb* pCallbackFn = (pTmq->commitCb != NULL) ? pTmq->commitCb : defaultCommitCbFn;
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
@ -1430,7 +1431,7 @@ void tmqHandleAllDelayedTask(tmq_t* pTmq) {
pTmq->autoCommitInterval / 1000.0); pTmq->autoCommitInterval / 1000.0);
bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, bool ret = taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer); &pTmq->commitTimer);
tqDebugC("reset timer fo commit:%d", ret); tqDebugC("reset timer for commit:%d", ret);
} else { } else {
tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); tqErrorC("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
} }

2
source/common/src/systable.c
View File

@ -118,7 +118,7 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "s3_chunkpages", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},

6
source/common/src/tglobal.c
View File

@ -287,7 +287,7 @@ int32_t tsTtlUnit = 86400;
int32_t tsTtlPushIntervalSec = 10; int32_t tsTtlPushIntervalSec = 10;
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups
bool tsS3MigrateEnabled = 1; bool tsS3MigrateEnabled = 0;
int32_t tsGrantHBInterval = 60; int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
@ -548,7 +548,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebug", tqClientDebug, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tqClientDebugFlag", tqClientDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER));
@ -2000,7 +2000,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
{"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},
{"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},
{"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag},
{"tqClientDebug", &tqClientDebug}, {"tqClientDebugFlag", &tqClientDebugFlag},
}; };
static OptionNameAndVar options[] = {{"audit", &tsEnableAudit}, static OptionNameAndVar options[] = {{"audit", &tsEnableAudit},

2
source/common/src/tmsg.c
View File

@ -10983,6 +10983,7 @@ _exit:
int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp)); TAOS_CHECK_RETURN(tEncodeMqDataRspCommon(pEncoder, pRsp));
TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime)); TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pRsp->sleepTime));
return 0; return 0;
} }
@ -11094,6 +11095,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen)); TAOS_CHECK_EXIT(tEncodeBinary(pEncoder, createTableReq, createTableLen));
} }
} }
_exit: _exit:
return code; return code;
} }

3
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -239,12 +239,13 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer)); MND_TMQ_RETURN_CHECK(mndAcquireConsumer(pMnode, consumerId, &pConsumer));
MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user)); MND_TMQ_RETURN_CHECK(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user));
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
mDebug("consumer:0x%" PRIx64 " receive hb pollFlag:%d %d", consumerId, req.pollFlag, pConsumer->pollStatus);
if (req.pollFlag == 1){ if (req.pollFlag == 1){
atomic_store_32(&pConsumer->pollStatus, 0); atomic_store_32(&pConsumer->pollStatus, 0);
} }
storeOffsetRows(pMnode, &req, pConsumer); storeOffsetRows(pMnode, &req, pConsumer);
rsp.debugFlag = tqClientDebug; rsp.debugFlag = tqClientDebugFlag;
code = buildMqHbRsp(pMsg, &rsp); code = buildMqHbRsp(pMsg, &rsp);
END: END:

2
source/dnode/mnode/impl/src/mndDb.c
View File

@ -583,7 +583,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE; if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL; if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT; if (pCfg->s3Compact < 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
if (pCfg->encryptAlgorithm < 0) pCfg->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO; if (pCfg->encryptAlgorithm < 0) pCfg->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
} }

32
source/dnode/mnode/impl/src/mndStb.c
View File

@ -4076,7 +4076,7 @@ typedef struct SMDropTbDbInfo {
typedef struct SMDropTbTsmaInfo { typedef struct SMDropTbTsmaInfo {
char tsmaResTbDbFName[TSDB_DB_FNAME_LEN]; char tsmaResTbDbFName[TSDB_DB_FNAME_LEN];
char tsmaResTbNamePrefix[TSDB_TABLE_NAME_LEN]; char tsmaResTbNamePrefix[TSDB_TABLE_FNAME_LEN];
int32_t suid; int32_t suid;
SMDropTbDbInfo dbInfo; // reference to DbInfo in pDbMap SMDropTbDbInfo dbInfo; // reference to DbInfo in pDbMap
} SMDropTbTsmaInfo; } SMDropTbTsmaInfo;
@ -4207,6 +4207,7 @@ static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg *pRsp, SMndDropTbsWithTsmaCtx
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs");
mndTransSetChangeless(pTrans); mndTransSetChangeless(pTrans);
mndTransSetSerial(pTrans);
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
@ -4294,6 +4295,18 @@ static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupI
return 0; return 0;
} }
int vgInfoCmp(const void* lp, const void* rp) {
SVgroupInfo* pLeft = (SVgroupInfo*)lp;
SVgroupInfo* pRight = (SVgroupInfo*)rp;
if (pLeft->hashBegin < pRight->hashBegin) {
return -1;
} else if (pLeft->hashBegin > pRight->hashBegin) {
return 1;
}
return 0;
}
static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropTbTsmaInfo *pInfo) { static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropTbTsmaInfo *pInfo) {
int32_t code = 0; int32_t code = 0;
SDbObj *pDb = mndAcquireDb(pMnode, dbname); SDbObj *pDb = mndAcquireDb(pMnode, dbname);
@ -4308,6 +4321,7 @@ static int32_t mndGetDbVgInfoForTsma(SMnode *pMnode, const char *dbname, SMDropT
goto _end; goto _end;
} }
mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos); mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos);
taosArraySort(pInfo->dbInfo.dbVgInfos, vgInfoCmp);
pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix; pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix;
pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix; pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix;
@ -4380,9 +4394,8 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith
if (pInfos) { if (pInfos) {
SMDropTbTsmaInfo info = {0}; SMDropTbTsmaInfo info = {0};
int32_t len = sprintf(buf, "%s", pSma->name); int32_t len = sprintf(buf, "%s", pSma->name);
len = taosCreateMD5Hash(buf, len);
sprintf(info.tsmaResTbDbFName, "%s", pSma->db); sprintf(info.tsmaResTbDbFName, "%s", pSma->db);
snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_NAME_LEN, "%s", buf); snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_FNAME_LEN, "%s", buf);
SMDropTbDbInfo *pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN); SMDropTbDbInfo *pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN);
info.suid = pSma->dstTbUid; info.suid = pSma->dstTbUid;
if (!pDbInfo) { if (!pDbInfo) {
@ -4417,14 +4430,17 @@ static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode *pMnode, SMndDropTbsWith
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid)); SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid));
SArray *pVgInfos = NULL; SArray *pVgInfos = NULL;
char buf[TSDB_TABLE_FNAME_LEN]; char buf[TSDB_TABLE_FNAME_LEN + TSDB_TABLE_NAME_LEN + 1];
char resTbFullName[TSDB_TABLE_FNAME_LEN + 1] = {0};
for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) { for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) {
SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j); SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j);
int32_t len = sprintf(buf, "%s.%s_%s", pInfo->tsmaResTbDbFName, pInfo->tsmaResTbNamePrefix, pTb->name); int32_t len = sprintf(buf, "%s_%s", pInfo->tsmaResTbNamePrefix, pTb->name);
uint32_t hashVal = len = taosCreateMD5Hash(buf, len);
taosGetTbHashVal(buf, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, pInfo->dbInfo.hashSuffix); len = snprintf(resTbFullName, TSDB_TABLE_FNAME_LEN + 1, "%s.%s", pInfo->tsmaResTbDbFName, buf);
uint32_t hashVal = taosGetTbHashVal(resTbFullName, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix,
pInfo->dbInfo.hashSuffix);
const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ); const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ);
void *p = taosStrdup(buf + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN); void *p = taosStrdup(resTbFullName + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN);
if (taosArrayPush(pCtx->pResTbNames, &p) == NULL) { if (taosArrayPush(pCtx->pResTbNames, &p) == NULL) {
code = terrno; code = terrno;
goto _end; goto _end;

2
source/dnode/vnode/inc/vnode.h
View File

@ -243,7 +243,7 @@ int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, con
int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, int64_t *createTime);
int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished); int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma // sma

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -163,7 +163,7 @@ int32_t metaDropTables(SMeta* pMeta, SArray* tbUids);
int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime);
int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock);

7
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -371,7 +371,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) {
return 0; return 0;
} }
SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) {
void *pData = NULL; void *pData = NULL;
int nData = 0; int nData = 0;
int64_t version; int64_t version;
@ -407,6 +407,9 @@ _query:
} }
} else if (me.type == TSDB_CHILD_TABLE) { } else if (me.type == TSDB_CHILD_TABLE) {
uid = me.ctbEntry.suid; uid = me.ctbEntry.suid;
if (createTime != NULL){
*createTime = me.ctbEntry.btime;
}
tDecoderClear(&dc); tDecoderClear(&dc);
goto _query; goto _query;
} else { } else {
@ -617,7 +620,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) {
STSchema *pTSchema = NULL; STSchema *pTSchema = NULL;
SSchemaWrapper *pSW = NULL; SSchemaWrapper *pSW = NULL;
pSW = metaGetTableSchema(pMeta, uid, sver, lock); pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL);
if (!pSW) return NULL; if (!pSW) return NULL;
pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version);

2
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -552,7 +552,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) {
void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }

12
source/dnode/vnode/src/tq/tqRead.c
View File

@ -263,7 +263,7 @@ bool tqGetTablePrimaryKey(STqReader* pReader) { return pReader->hasPrimaryKey; }
void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) {
bool ret = false; bool ret = false;
SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL);
if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) {
ret = true; ret = true;
} }
@ -669,7 +669,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
(pReader->cachedSchemaVer != sversion)) { (pReader->cachedSchemaVer != sversion)) {
tDeleteSchemaWrapper(pReader->pSchemaWrapper); tDeleteSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL);
if (pReader->pSchemaWrapper == NULL) { if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64
"version %d, possibly dropped table", "version %d, possibly dropped table",
@ -961,10 +961,8 @@ END:
return code; return code;
} }
int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) {
tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); tqTrace("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk);
SSDataBlock* block = NULL;
SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
if (pSubmitTbData == NULL) { if (pSubmitTbData == NULL) {
return terrno; return terrno;
@ -980,7 +978,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
pReader->lastBlkUid = uid; pReader->lastBlkUid = uid;
tDeleteSchemaWrapper(pReader->pSchemaWrapper); tDeleteSchemaWrapper(pReader->pSchemaWrapper);
pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime);
if (pReader->pSchemaWrapper == NULL) { if (pReader->pSchemaWrapper == NULL) {
tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer);

140
source/dnode/vnode/src/tq/tqScan.c
View File

@ -210,36 +210,21 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pDataBlock != NULL && pDataBlock->info.rows > 0) {
if (pRsp->withTbName) { if (pRsp->withTbName) {
if (pOffset->type == TMQ_OFFSET__LOG) { char* tbName = taosStrdup(qExtractTbnameFromTask(task));
int64_t uid = pExec->pTqReader->lastBlkUid; if (tbName == NULL) {
if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId);
tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId); return terrno;
continue; }
} if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){
} else { tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
char* tbName = taosStrdup(qExtractTbnameFromTask(task)); continue;
if (tbName == NULL) {
tqError("vgId:%d, failed to add tbname to rsp msg, null", pTq->pVnode->config.vgId);
return terrno;
}
if (taosArrayPush(pRsp->blockTbName, &tbName) == NULL){
tqError("vgId:%d, failed to add tbname to rsp msg", pTq->pVnode->config.vgId);
continue;
}
} }
} }
if (pRsp->withSchema) { if (pRsp->withSchema) {
if (pOffset->type == TMQ_OFFSET__LOG) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
if (tqAddBlockSchemaToRsp(pExec, pRsp) != 0){ if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
continue; continue;
}
} else {
SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task));
if(taosArrayPush(pRsp->blockSchema, &pSW) == NULL){
tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId);
continue;
}
} }
} }
@ -249,12 +234,9 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
continue; continue;
} }
pRsp->blockNum++; pRsp->blockNum++;
if (pOffset->type == TMQ_OFFSET__LOG) { rowCnt += pDataBlock->info.rows;
continue; if (rowCnt <= tmqRowSize) continue;
} else {
rowCnt += pDataBlock->info.rows;
if (rowCnt <= tmqRowSize) continue;
}
} }
// get meta // get meta
@ -296,6 +278,54 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat
return code; return code;
} }
static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){
int32_t code = 0;
void* createReq = NULL;
if (pRsp->createTableNum == 0) {
pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t));
if (pRsp->createTableLen == NULL) {
code = terrno;
goto END;
}
pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
if (pRsp->createTableReq == NULL) {
code = terrno;
goto END;
}
}
uint32_t len = 0;
tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code);
if (TSDB_CODE_SUCCESS != code) {
goto END;
}
createReq = taosMemoryCalloc(1, len);
if (createReq == NULL){
code = terrno;
goto END;
}
SEncoder encoder = {0};
tEncoderInit(&encoder, createReq, len);
code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq);
tEncoderClear(&encoder);
if (code < 0) {
goto END;
}
if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
code = terrno;
goto END;
}
if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
code = terrno;
goto END;
}
pRsp->createTableNum++;
return 0;
END:
taosMemoryFree(createReq);
return code;
}
static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){
int32_t code = 0; int32_t code = 0;
@ -315,7 +345,8 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
} }
SSubmitTbData* pSubmitTbDataRet = NULL; SSubmitTbData* pSubmitTbDataRet = NULL;
code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet); int64_t createTime = INT64_MAX;
code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime);
if (code != 0) { if (code != 0) {
tqError("vgId:%d, failed to retrieve block", pTq->pVnode->config.vgId); tqError("vgId:%d, failed to retrieve block", pTq->pVnode->config.vgId);
goto END; goto END;
@ -333,46 +364,13 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int
} }
} }
if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) {
if (pRsp->createTableNum == 0) { if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) { // judge if table is already created to avoid sending crateTbReq
pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq);
if (pRsp->createTableLen == NULL) { if (code != 0){
code = terrno; tqError("vgId:%d, failed to build create table info", pTq->pVnode->config.vgId);
goto END;
}
pRsp->createTableReq = taosArrayInit(0, sizeof(void*));
if (pRsp->createTableReq == NULL) {
code = terrno;
goto END; goto END;
} }
} }
uint32_t len = 0;
tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code);
if (TSDB_CODE_SUCCESS != code) {
goto END;
}
void* createReq = taosMemoryCalloc(1, len);
if (createReq == NULL){
code = terrno;
goto END;
}
SEncoder encoder = {0};
tEncoderInit(&encoder, createReq, len);
code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq);
tEncoderClear(&encoder);
if (code < 0) {
taosMemoryFree(createReq);
goto END;
}
if (taosArrayPush(pRsp->createTableLen, &len) == NULL){
taosMemoryFree(createReq);
goto END;
}
if (taosArrayPush(pRsp->createTableReq, &createReq) == NULL){
taosMemoryFree(createReq);
goto END;
}
pRsp->createTableNum++;
} }
if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) { if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) {
goto END; goto END;

4
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -51,7 +51,8 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); pRsp->blockTbName = taosArrayInit(0, sizeof(void*));
pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); pRsp->blockSchema = taosArrayInit(0, sizeof(void*));
if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL ||
pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) {
if (pRsp->blockData != NULL) { if (pRsp->blockData != NULL) {
taosArrayDestroy(pRsp->blockData); taosArrayDestroy(pRsp->blockData);
pRsp->blockData = NULL; pRsp->blockData = NULL;
@ -71,6 +72,7 @@ static int32_t tqInitTaosxRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) {
taosArrayDestroy(pRsp->blockSchema); taosArrayDestroy(pRsp->blockSchema);
pRsp->blockSchema = NULL; pRsp->blockSchema = NULL;
} }
return terrno; return terrno;
} }

7
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -409,7 +409,12 @@ int32_t vnodeSyncCommit(SVnode *pVnode) {
vnodeAWait(&pVnode->commitTask); vnodeAWait(&pVnode->commitTask);
_exit: _exit:
vError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); if (code) {
vError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
} else {
vInfo("vgId:%d, sync commit end", TD_VID(pVnode));
}
return code; return code;
} }

2
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -702,7 +702,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) {
} }
int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0); SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL);
if (pSW) { if (pSW) {
*num = pSW->nCols; *num = pSW->nCols;
tDeleteSchemaWrapper(pSW); tDeleteSchemaWrapper(pSW);

19
source/libs/azure/src/az.cpp
View File

@ -416,7 +416,7 @@ _exit:
return 0; return 0;
} }
int32_t azGetObjectToFile(const char *object_name, const char *fileName) { static int32_t azGetObjectToFileImpl(const char *object_name, const char *fileName) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
std::string accountName = tsS3AccessKeyId[0]; std::string accountName = tsS3AccessKeyId[0];
std::string accountKey = tsS3AccessKeySecret[0]; std::string accountKey = tsS3AccessKeySecret[0];
@ -450,6 +450,23 @@ int32_t azGetObjectToFile(const char *object_name, const char *fileName) {
TAOS_RETURN(code); TAOS_RETURN(code);
} }
int32_t azGetObjectToFile(const char *object_name, const char *fileName) {
int32_t code = 0;
try {
code = azGetObjectToFileImpl(object_name, fileName);
} catch (const std::exception &e) {
azError("%s: Reason Phrase: %s", __func__, e.what());
code = TAOS_SYSTEM_ERROR(EIO);
azError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
TAOS_RETURN(code);
}
TAOS_RETURN(code);
}
int32_t azGetObjectsByPrefix(const char *prefix, const char *path) { int32_t azGetObjectsByPrefix(const char *prefix, const char *path) {
const std::string delimiter = "/"; const std::string delimiter = "/";
std::string accountName = tsS3AccessKeyId[0]; std::string accountName = tsS3AccessKeyId[0];

146
source/libs/command/src/command.c
View File

@ -50,7 +50,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
(*pRsp)->numOfCols = htonl(numOfCols); (*pRsp)->numOfCols = htonl(numOfCols);
int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols);
if(len < 0) { if (len < 0) {
taosMemoryFree(*pRsp); taosMemoryFree(*pRsp);
return terrno; return terrno;
} }
@ -292,7 +292,7 @@ static int32_t buildRetension(SArray* pRetension, char** ppRetentions) {
} }
const int lMaxLen = 128; const int lMaxLen = 128;
char* p1 = taosMemoryCalloc(1, lMaxLen); char* p1 = taosMemoryCalloc(1, lMaxLen);
if (NULL == p1) { if (NULL == p1) {
return terrno; return terrno;
} }
@ -346,20 +346,20 @@ static const char* encryptAlgorithmStr(int8_t encryptAlgorithm) {
} }
int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes) { int32_t formatDurationOrKeep(char* buffer, int64_t bufSize, int32_t timeInMinutes) {
if (buffer == NULL || bufSize <= 0) { if (buffer == NULL || bufSize <= 0) {
return 0; return 0;
} }
int32_t len = 0; int32_t len = 0;
if (timeInMinutes % 1440 == 0) { if (timeInMinutes % 1440 == 0) {
int32_t days = timeInMinutes / 1440; int32_t days = timeInMinutes / 1440;
len = tsnprintf(buffer, bufSize, "%dd", days); len = tsnprintf(buffer, bufSize, "%dd", days);
} else if (timeInMinutes % 60 == 0) { } else if (timeInMinutes % 60 == 0) {
int32_t hours = timeInMinutes / 60; int32_t hours = timeInMinutes / 60;
len = tsnprintf(buffer, bufSize, "%dh", hours); len = tsnprintf(buffer, bufSize, "%dh", hours);
} else { } else {
len = tsnprintf(buffer, bufSize, "%dm", timeInMinutes); len = tsnprintf(buffer, bufSize, "%dm", timeInMinutes);
} }
return len; return len;
} }
static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) { static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, char* dbFName, SDbCfgInfo* pCfg) {
@ -410,27 +410,27 @@ static int32_t setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName,
int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2); int32_t lenKeep2 = formatDurationOrKeep(keep2Str, sizeof(keep2Str), pCfg->daysToKeep2);
if (IS_SYS_DBNAME(dbName)) { if (IS_SYS_DBNAME(dbName)) {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE DATABASE `%s`", dbName);
} else {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s " "CREATE DATABASE `%s`", dbName);
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d " } else {
"PRECISION '%s' REPLICA %d " len +=
"WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d " tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64 "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %s "
" KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKSIZE %d S3_KEEPLOCAL %dm S3_COMPACT %d", "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d STT_TRIGGER %d KEEP %s,%s,%s PAGES %d PAGESIZE %d "
dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, "PRECISION '%s' REPLICA %d "
durationStr, "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d TABLE_PREFIX %d TABLE_SUFFIX %d TSDB_PAGESIZE %d "
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, "WAL_RETENTION_PERIOD %d WAL_RETENTION_SIZE %" PRId64
keep0Str, keep1Str, keep2Str, " KEEP_TIME_OFFSET %d ENCRYPT_ALGORITHM '%s' S3_CHUNKPAGES %d S3_KEEPLOCAL %dm S3_COMPACT %d",
pCfg->pages, pCfg->pageSize, prec, dbName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, durationStr,
pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups, 1 == pCfg->numOfStables, hashPrefix, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->sstTrigger, keep0Str, keep1Str, keep2Str,
pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod, pCfg->walRetentionSize, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm), pCfg->s3ChunkSize, 1 == pCfg->numOfStables, hashPrefix, pCfg->hashSuffix, pCfg->tsdbPageSize, pCfg->walRetentionPeriod,
pCfg->s3KeepLocal, pCfg->s3Compact); pCfg->walRetentionSize, pCfg->keepTimeOffset, encryptAlgorithmStr(pCfg->encryptAlgorithm),
pCfg->s3ChunkSize, pCfg->s3KeepLocal, pCfg->s3Compact);
if (pRetentions) { if (pRetentions) {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, " RETENTIONS %s", pRetentions); len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_DB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
" RETENTIONS %s", pRetentions);
} }
} }
@ -510,30 +510,30 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
#define LTYPE_LEN (32 + 60) // 60 byte for compress info #define LTYPE_LEN (32 + 60) // 60 byte for compress info
char type[LTYPE_LEN]; char type[LTYPE_LEN];
snprintf(type, LTYPE_LEN, "%s", tDataTypes[pSchema->type].name); snprintf(type, LTYPE_LEN, "%s", tDataTypes[pSchema->type].name);
int typeLen = strlen(type); int typeLen = strlen(type);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type ||
TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)", typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} }
if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) { if (useCompress(pCfg->tableType) && pCfg->pSchemaExt) {
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'", typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " ENCODE \'%s\'",
columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress))); columnEncodeStr(COMPRESS_L1_TYPE_U32(pCfg->pSchemaExt[i].compress)));
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'", typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " COMPRESS \'%s\'",
columnCompressStr(COMPRESS_L2_TYPE_U32(pCfg->pSchemaExt[i].compress))); columnCompressStr(COMPRESS_L2_TYPE_U32(pCfg->pSchemaExt[i].compress)));
typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " LEVEL \'%s\'", typeLen += tsnprintf(type + typeLen, LTYPE_LEN - typeLen, " LEVEL \'%s\'",
columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress))); columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pCfg->pSchemaExt[i].compress)));
} }
if (!(pSchema->flags & COL_IS_KEY)) { if (!(pSchema->flags & COL_IS_KEY)) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
((i > 0) ? ", " : ""), pSchema->name, type); "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
} else { } else {
char* pk = "PRIMARY KEY"; char* pk = "PRIMARY KEY";
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s %s", *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
((i > 0) ? ", " : ""), pSchema->name, type, pk); "%s`%s` %s %s", ((i > 0) ? ", " : ""), pSchema->name, type, pk);
} }
} }
} }
@ -545,14 +545,15 @@ void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) {
snprintf(type, sizeof(type), "%s", tDataTypes[pSchema->type].name); snprintf(type, sizeof(type), "%s", tDataTypes[pSchema->type].name);
if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type || if (TSDB_DATA_TYPE_VARCHAR == pSchema->type || TSDB_DATA_TYPE_VARBINARY == pSchema->type ||
TSDB_DATA_TYPE_GEOMETRY == pSchema->type) { TSDB_DATA_TYPE_GEOMETRY == pSchema->type) {
snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)",
(int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE));
} else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) {
snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)", snprintf(type + strlen(type), sizeof(type) - strlen(type), "(%d)",
(int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} }
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "%s`%s` %s", *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
((i > 0) ? ", " : ""), pSchema->name, type); "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type);
} }
} }
@ -560,7 +561,7 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfTags; ++i) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name);
} }
} }
@ -582,7 +583,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
return terrno; return terrno;
} }
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s", pJson); "%s", pJson);
taosMemoryFree(pJson); taosMemoryFree(pJson);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -596,12 +597,12 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i;
if (i > 0) { if (i > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", "); ", ");
} }
if (j >= valueNum) { if (j >= valueNum) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"NULL"); "NULL");
continue; continue;
} }
@ -624,14 +625,15 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, pTagVal->pData, pTagVal->nData, &tlen); code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, pTagVal->pData, pTagVal->nData, &tlen);
TAOS_CHECK_ERRNO(code); TAOS_CHECK_ERRNO(code);
} else { } else {
code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); code = dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, leftSize, type, &pTagVal->i64, tDataTypes[type].bytes,
&tlen);
TAOS_CHECK_ERRNO(code); TAOS_CHECK_ERRNO(code);
} }
*len += tlen; *len += tlen;
j++; j++;
} else { } else {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"NULL"); "NULL");
} }
} }
_exit: _exit:
@ -643,38 +645,38 @@ _exit:
void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) { void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg* pCfg) {
if (pCfg->commentLen > 0) { if (pCfg->commentLen > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" COMMENT '%s'", pCfg->pComment); " COMMENT '%s'", pCfg->pComment);
} else if (0 == pCfg->commentLen) { } else if (0 == pCfg->commentLen) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" COMMENT ''"); " COMMENT ''");
} }
if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) { if (NULL != pDbCfg->pRetensions && pCfg->watermark1 > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" WATERMARK %" PRId64 "a", pCfg->watermark1); " WATERMARK %" PRId64 "a", pCfg->watermark1);
if (pCfg->watermark2 > 0) { if (pCfg->watermark2 > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", %" PRId64 "a", pCfg->watermark2); ", %" PRId64 "a", pCfg->watermark2);
} }
} }
if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) { if (NULL != pDbCfg->pRetensions && pCfg->delay1 > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" MAX_DELAY %" PRId64 "a", pCfg->delay1); " MAX_DELAY %" PRId64 "a", pCfg->delay1);
if (pCfg->delay2 > 0) { if (pCfg->delay2 > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
", %" PRId64 "a", pCfg->delay2); ", %" PRId64 "a", pCfg->delay2);
} }
} }
int32_t funcNum = taosArrayGetSize(pCfg->pFuncs); int32_t funcNum = taosArrayGetSize(pCfg->pFuncs);
if (NULL != pDbCfg->pRetensions && funcNum > 0) { if (NULL != pDbCfg->pRetensions && funcNum > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" ROLLUP("); " ROLLUP(");
for (int32_t i = 0; i < funcNum; ++i) { for (int32_t i = 0; i < funcNum; ++i) {
char* pFunc = taosArrayGet(pCfg->pFuncs, i); char* pFunc = taosArrayGet(pCfg->pFuncs, i);
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
"%s%s", ((i > 0) ? ", " : ""), pFunc); "%s%s", ((i > 0) ? ", " : ""), pFunc);
} }
*len += *len +=
snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ")"); snprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ")");
@ -682,7 +684,7 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg*
if (pCfg->ttl > 0) { if (pCfg->ttl > 0) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" TTL %d", pCfg->ttl); " TTL %d", pCfg->ttl);
} }
if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) { if (TSDB_SUPER_TABLE == pCfg->tableType || TSDB_NORMAL_TABLE == pCfg->tableType) {
@ -696,18 +698,18 @@ void appendTableOptions(char* buf, int32_t* len, SDbCfgInfo* pDbCfg, STableCfg*
if (nSma < pCfg->numOfColumns && nSma > 0) { if (nSma < pCfg->numOfColumns && nSma > 0) {
bool smaOn = false; bool smaOn = false;
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len),
" SMA("); " SMA(");
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
if (IS_BSMA_ON(pCfg->pSchemas + i)) { if (IS_BSMA_ON(pCfg->pSchemas + i)) {
if (smaOn) { if (smaOn) {
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len,
SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ",`%s`", SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), ",`%s`",
(pCfg->pSchemas + i)->name); (pCfg->pSchemas + i)->name);
} else { } else {
smaOn = true; smaOn = true;
*len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len, *len += tsnprintf(buf + VARSTR_HEADER_SIZE + *len,
SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "`%s`", SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + *len), "`%s`",
(pCfg->pSchemas + i)->name); (pCfg->pSchemas + i)->name);
} }
} }
} }
@ -736,20 +738,20 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
if (TSDB_SUPER_TABLE == pCfg->tableType) { if (TSDB_SUPER_TABLE == pCfg->tableType) {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE STABLE `%s` (", tbName); "CREATE STABLE `%s` (", tbName);
appendColumnFields(buf2, &len, pCfg); appendColumnFields(buf2, &len, pCfg);
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len),
") TAGS ("); ") TAGS (");
appendTagFields(buf2, &len, pCfg); appendTagFields(buf2, &len, pCfg);
len += len +=
snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")");
appendTableOptions(buf2, &len, pDbCfg, pCfg); appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else if (TSDB_CHILD_TABLE == pCfg->tableType) { } else if (TSDB_CHILD_TABLE == pCfg->tableType) {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName);
appendTagNameFields(buf2, &len, pCfg); appendTagNameFields(buf2, &len, pCfg);
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), len += tsnprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len),
") TAGS ("); ") TAGS (");
code = appendTagValues(buf2, &len, pCfg); code = appendTagValues(buf2, &len, pCfg);
TAOS_CHECK_ERRNO(code); TAOS_CHECK_ERRNO(code);
len += len +=
@ -757,7 +759,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
appendTableOptions(buf2, &len, pDbCfg, pCfg); appendTableOptions(buf2, &len, pDbCfg, pCfg);
} else { } else {
len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, len += tsnprintf(buf2 + VARSTR_HEADER_SIZE, SHOW_CREATE_TB_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE,
"CREATE TABLE `%s` (", tbName); "CREATE TABLE `%s` (", tbName);
appendColumnFields(buf2, &len, pCfg); appendColumnFields(buf2, &len, pCfg);
len += len +=
snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")"); snprintf(buf2 + VARSTR_HEADER_SIZE + len, SHOW_CREATE_TB_RESULT_FIELD2_LEN - (VARSTR_HEADER_SIZE + len), ")");
@ -793,7 +795,7 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate
} }
SViewMeta* pMeta = pStmt->pViewMeta; SViewMeta* pMeta = pStmt->pViewMeta;
if(NULL == pMeta) { if (NULL == pMeta) {
qError("exception: view meta is null"); qError("exception: view meta is null");
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }

1
source/libs/executor/src/executor.c
View File

@ -1495,6 +1495,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond);
tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN); tstrncpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName, TSDB_TABLE_NAME_LEN);
// pTaskInfo->streamInfo.suid = mtInfo.suid == 0 ? mtInfo.uid : mtInfo.suid;
tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema); tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema);
pTaskInfo->streamInfo.schema = mtInfo.schema; pTaskInfo->streamInfo.schema = mtInfo.schema;

4
source/libs/parser/inc/parAst.h
View File

@ -64,7 +64,7 @@ typedef enum EDatabaseOptionType {
DB_OPTION_STT_TRIGGER, DB_OPTION_STT_TRIGGER,
DB_OPTION_TABLE_PREFIX, DB_OPTION_TABLE_PREFIX,
DB_OPTION_TABLE_SUFFIX, DB_OPTION_TABLE_SUFFIX,
DB_OPTION_S3_CHUNKSIZE, DB_OPTION_S3_CHUNKPAGES,
DB_OPTION_S3_KEEPLOCAL, DB_OPTION_S3_KEEPLOCAL,
DB_OPTION_S3_COMPACT, DB_OPTION_S3_COMPACT,
DB_OPTION_KEEP_TIME_OFFSET, DB_OPTION_KEEP_TIME_OFFSET,
@ -244,7 +244,7 @@ SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable
SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern);
SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint); SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint);
SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* pDbName, SNodeList* pTags); SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* pDbName, SNodeList* pTags);
SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo, SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo,
int8_t createdb, int8_t is_import); int8_t createdb, int8_t is_import);
SNode* addCreateUserStmtWhiteList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pIpRangesNodeList); SNode* addCreateUserStmtWhiteList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pIpRangesNodeList);
SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, void* pAlterInfo); SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, void* pAlterInfo);

2
source/libs/parser/inc/sql.y
View File

@ -280,7 +280,7 @@ db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); } db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); } db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
db_options(A) ::= db_options(B) S3_CHUNKSIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKSIZE, &C); } db_options(A) ::= db_options(B) S3_CHUNKPAGES NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_CHUNKPAGES, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); } db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); } db_options(A) ::= db_options(B) S3_KEEPLOCAL NK_VARIABLE(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_KEEPLOCAL, &C); }
db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); } db_options(A) ::= db_options(B) S3_COMPACT NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_S3_COMPACT, &C); }

28
source/libs/parser/src/parAstCreater.c
View File

@ -43,11 +43,11 @@
} \ } \
} while (0) } while (0)
#define CHECK_NAME(p) \ #define CHECK_NAME(p) \
do { \ do { \
if (!p) { \ if (!p) { \
goto _err; \ goto _err; \
} \ } \
} while (0) } while (0)
#define COPY_STRING_FORM_ID_TOKEN(buf, pToken) strncpy(buf, (pToken)->z, TMIN((pToken)->n, sizeof(buf) - 1)) #define COPY_STRING_FORM_ID_TOKEN(buf, pToken) strncpy(buf, (pToken)->z, TMIN((pToken)->n, sizeof(buf) - 1))
@ -333,7 +333,7 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) {
// Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN]. // Len of pRawExpr->p could be larger than len of aliasName[TSDB_COL_NAME_LEN].
// If aliasName is truncated, hash value of aliasName could be the same. // If aliasName is truncated, hash value of aliasName could be the same.
uint64_t hashVal = MurmurHash3_64(pRawExpr->p, pRawExpr->n); uint64_t hashVal = MurmurHash3_64(pRawExpr->p, pRawExpr->n);
sprintf(pExpr->aliasName, "%"PRIu64, hashVal); sprintf(pExpr->aliasName, "%" PRIu64, hashVal);
strncpy(pExpr->userAlias, pRawExpr->p, len); strncpy(pExpr->userAlias, pRawExpr->p, len);
pExpr->userAlias[len] = '\0'; pExpr->userAlias[len] = '\0';
} }
@ -405,7 +405,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
pCxt->errCode = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&val); pCxt->errCode = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&val);
CHECK_MAKE_NODE(val); CHECK_MAKE_NODE(val);
val->literal = taosStrndup(pLiteral->z, pLiteral->n); val->literal = taosStrndup(pLiteral->z, pLiteral->n);
if(!val->literal) { if (!val->literal) {
pCxt->errCode = terrno; pCxt->errCode = terrno;
nodesDestroyNode((SNode*)val); nodesDestroyNode((SNode*)val);
return NULL; return NULL;
@ -586,8 +586,8 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) {
if (NULL == pLiteral || pLiteral->n <= 5) { if (NULL == pLiteral || pLiteral->n <= 5) {
return NULL; return NULL;
} }
SNodeList* pHintList = NULL; SNodeList* pHintList = NULL;
char* hint = taosStrndup(pLiteral->z + 3, pLiteral->n - 5); char* hint = taosStrndup(pLiteral->z + 3, pLiteral->n - 5);
if (!hint) return NULL; if (!hint) return NULL;
int32_t i = 0; int32_t i = 0;
bool quit = false; bool quit = false;
@ -971,7 +971,7 @@ _err:
} }
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) {
SNode* pNew = NULL, *pGE = NULL, *pLE = NULL; SNode *pNew = NULL, *pGE = NULL, *pLE = NULL;
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesCloneNode(pExpr, &pNew); pCxt->errCode = nodesCloneNode(pExpr, &pNew);
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
@ -993,7 +993,7 @@ _err:
} }
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) {
SNode* pNew = NULL, *pLT = NULL, *pGT = NULL; SNode *pNew = NULL, *pLT = NULL, *pGT = NULL;
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesCloneNode(pExpr, &pNew); pCxt->errCode = nodesCloneNode(pExpr, &pNew);
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
@ -1959,7 +1959,7 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
nodesDestroyNode((SNode*)pNode); nodesDestroyNode((SNode*)pNode);
break; break;
} }
case DB_OPTION_S3_CHUNKSIZE: case DB_OPTION_S3_CHUNKPAGES:
pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); pDbOptions->s3ChunkSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break; break;
case DB_OPTION_S3_KEEPLOCAL: { case DB_OPTION_S3_KEEPLOCAL: {
@ -2209,7 +2209,7 @@ _err:
SNode* setColumnOptions(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal1, void* pVal2) { SNode* setColumnOptions(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal1, void* pVal2) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
char optionType[TSDB_CL_OPTION_LEN]; char optionType[TSDB_CL_OPTION_LEN];
memset(optionType, 0, TSDB_CL_OPTION_LEN); memset(optionType, 0, TSDB_CL_OPTION_LEN);
strncpy(optionType, pVal1->z, TMIN(pVal1->n, TSDB_CL_OPTION_LEN)); strncpy(optionType, pVal1->z, TMIN(pVal1->n, TSDB_CL_OPTION_LEN));
@ -2806,7 +2806,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
char* ipCopy = taosStrdup(ipRange); char* ipCopy = taosStrdup(ipRange);
if (!ipCopy) return terrno; if (!ipCopy) return terrno;
char* slash = strchr(ipCopy, '/'); char* slash = strchr(ipCopy, '/');
if (slash) { if (slash) {
*slash = '\0'; *slash = '\0';
struct in_addr addr; struct in_addr addr;

47
source/libs/parser/src/parInsertUtil.c
View File

@ -886,17 +886,32 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
return false; return false;
} }
int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* tFields,
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen) { int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw) {
int ret = 0;
if(data == NULL) { if(data == NULL) {
uError("rawBlockBindData, data is NULL"); uError("rawBlockBindData, data is NULL");
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
} }
void* tmp = void* tmp =
taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid));
SVCreateTbReq *pCreateReqTmp = NULL;
if (tmp == NULL && pCreateTb != NULL){
ret = cloneSVreateTbReq(pCreateTb, &pCreateReqTmp);
if (ret != TSDB_CODE_SUCCESS){
uError("cloneSVreateTbReq error");
goto end;
}
}
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); sizeof(pTableMeta->uid), pTableMeta, &pCreateReqTmp, &pTableCxt, true, false);
if (pCreateReqTmp != NULL) {
tdDestroySVCreateTbReq(pCreateReqTmp);
taosMemoryFree(pCreateReqTmp);
}
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("insGetTableDataCxt error"); uError("insGetTableDataCxt error");
goto end; goto end;
@ -948,12 +963,17 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
if (tFields != NULL && numFields > boundInfo->numOfBound) { // if (tFields != NULL && numFields > boundInfo->numOfBound) {
if (errstr != NULL) // if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound); // ret = TSDB_CODE_INVALID_PARA;
// goto end;
// }
if (tFields == NULL && numOfCols != boundInfo->numOfBound) {
if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to num of bound cols:%d", numOfCols, boundInfo->numOfBound);
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
if (tFields == NULL) { if (tFields == NULL) {
for (int j = 0; j < boundInfo->numOfBound; j++) { for (int j = 0; j < boundInfo->numOfBound; j++) {
SSchema* pColSchema = &pSchema[j]; SSchema* pColSchema = &pSchema[j];
@ -991,7 +1011,13 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
for (int i = 0; i < numFields; i++) { for (int i = 0; i < numFields; i++) {
for (int j = 0; j < boundInfo->numOfBound; j++) { for (int j = 0; j < boundInfo->numOfBound; j++) {
SSchema* pColSchema = &pSchema[j]; SSchema* pColSchema = &pSchema[j];
if (strcmp(pColSchema->name, tFields[i].name) == 0) { char* fieldName = NULL;
if (raw) {
fieldName = ((SSchemaWrapper*)tFields)->pSchema[i].name;
} else {
fieldName = ((TAOS_FIELD*)tFields)[i].name;
}
if (strcmp(pColSchema->name, fieldName) == 0) {
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
if (errstr != NULL) if (errstr != NULL)
snprintf(errstr, errstrLen, snprintf(errstr, errstrLen,
@ -1011,6 +1037,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
pStart += numOfRows * sizeof(int32_t); pStart += numOfRows * sizeof(int32_t);
} else { } else {
pStart += BitmapLen(numOfRows); pStart += BitmapLen(numOfRows);
// for(int k = 0; k < numOfRows; k++) {
// if(!colDataIsNull_f(offset, k) && pColSchema->type == TSDB_DATA_TYPE_INT){
// printf("colName:%s,val:%d", fieldName, *(int32_t*)(pStart + k * sizeof(int32_t)));
// }
// }
} }
char* pData = pStart; char* pData = pStart;

14
source/libs/parser/src/parTokenizer.c
View File

@ -340,7 +340,7 @@ static SKeyword keywordTable[] = {
{"_FROWTS", TK_FROWTS}, {"_FROWTS", TK_FROWTS},
{"ALIVE", TK_ALIVE}, {"ALIVE", TK_ALIVE},
{"VARBINARY", TK_VARBINARY}, {"VARBINARY", TK_VARBINARY},
{"S3_CHUNKSIZE", TK_S3_CHUNKSIZE}, {"S3_CHUNKPAGES", TK_S3_CHUNKPAGES},
{"S3_KEEPLOCAL", TK_S3_KEEPLOCAL}, {"S3_KEEPLOCAL", TK_S3_KEEPLOCAL},
{"S3_COMPACT", TK_S3_COMPACT}, {"S3_COMPACT", TK_S3_COMPACT},
{"S3MIGRATE", TK_S3MIGRATE}, {"S3MIGRATE", TK_S3MIGRATE},
@ -370,7 +370,7 @@ static int32_t doInitKeywordsTable(void) {
keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); keywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false);
for (int32_t i = 0; i < numOfEntries; i++) { for (int32_t i = 0; i < numOfEntries; i++) {
keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name); keywordTable[i].len = (uint8_t)strlen(keywordTable[i].name);
void* ptr = &keywordTable[i]; void* ptr = &keywordTable[i];
int32_t code = taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES); int32_t code = taosHashPut(keywordHashTable, keywordTable[i].name, keywordTable[i].len, (void*)&ptr, POINTER_BYTES);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
taosHashCleanup(keywordHashTable); taosHashCleanup(keywordHashTable);
@ -698,7 +698,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
} }
} }
if (hasNonAsciiChars) { if (hasNonAsciiChars) {
*tokenId = TK_NK_ALIAS; // must be alias *tokenId = TK_NK_ALIAS; // must be alias
return i; return i;
} }
if (IS_TRUE_STR(z, i) || IS_FALSE_STR(z, i)) { if (IS_TRUE_STR(z, i) || IS_FALSE_STR(z, i)) {
@ -713,10 +713,10 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
break; break;
} }
bool hasNonAsciiChars = false; bool hasNonAsciiChars = false;
for (i = 1; ; i++) { for (i = 1;; i++) {
if ((z[i] & 0x80) != 0) { if ((z[i] & 0x80) != 0) {
hasNonAsciiChars = true; hasNonAsciiChars = true;
} else if (isIdChar[(uint8_t)z[i]]){ } else if (isIdChar[(uint8_t)z[i]]) {
} else { } else {
break; break;
} }
@ -834,9 +834,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC
bool taosIsKeyWordToken(const char* z, int32_t len) { return (tKeywordCode((char*)z, len) != TK_NK_ID); } bool taosIsKeyWordToken(const char* z, int32_t len) { return (tKeywordCode((char*)z, len) != TK_NK_ID); }
int32_t taosInitKeywordsTable() { int32_t taosInitKeywordsTable() { return doInitKeywordsTable(); }
return doInitKeywordsTable();
}
void taosCleanupKeywordsTable() { void taosCleanupKeywordsTable() {
void* m = keywordHashTable; void* m = keywordHashTable;

503
source/libs/parser/src/parTranslater.c
View File

File diff suppressed because it is too large Load Diff

1502
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

40
source/libs/scalar/src/filter.c
View File

@ -4133,7 +4133,7 @@ int32_t fltSclBuildDatumFromValueNode(SFltSclDatum *datum, SValueNode *valNode)
} }
case TSDB_DATA_TYPE_BOOL: { case TSDB_DATA_TYPE_BOOL: {
datum->kind = FLT_SCL_DATUM_KIND_INT64; datum->kind = FLT_SCL_DATUM_KIND_INT64;
datum->i = (valNode->datum.b) ? 0 : 1; datum->i = (valNode->datum.b) ? 1 : 0;
break; break;
} }
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
@ -4541,6 +4541,7 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) {
if (info->scalarMode) { if (info->scalarMode) {
SArray *colRanges = info->sclCtx.fltSclRange; SArray *colRanges = info->sclCtx.fltSclRange;
SOperatorNode *optNode = (SOperatorNode *) pNode;
if (taosArrayGetSize(colRanges) == 1) { if (taosArrayGetSize(colRanges) == 1) {
SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0); SFltSclColumnRange *colRange = taosArrayGet(colRanges, 0);
if (NULL == colRange) { if (NULL == colRange) {
@ -4560,7 +4561,8 @@ int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict) {
FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end)); FLT_ERR_JRET(fltSclGetTimeStampDatum(endPt, &end));
win->skey = start.i; win->skey = start.i;
win->ekey = end.i; win->ekey = end.i;
*isStrict = true; if(optNode->opType == OP_TYPE_IN) *isStrict = false;
else *isStrict = true;
goto _return; goto _return;
} else if (taosArrayGetSize(points) == 0) { } else if (taosArrayGetSize(points) == 0) {
*win = TSWINDOW_DESC_INITIALIZER; *win = TSWINDOW_DESC_INITIALIZER;
@ -5023,6 +5025,34 @@ int32_t fltSclBuildRangePoints(SFltSclOperator *oper, SArray *points) {
} }
break; break;
} }
case OP_TYPE_IN: {
SNodeListNode *listNode = (SNodeListNode *)oper->valNode;
SListCell *cell = listNode->pNodeList->pHead;
SFltSclDatum minDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MAX, .type = oper->colNode->node.resType};
SFltSclDatum maxDatum = {.kind = FLT_SCL_DATUM_KIND_INT64, .i = INT64_MIN, .type = oper->colNode->node.resType};
for (int32_t i = 0; i < listNode->pNodeList->length; ++i) {
SValueNode *valueNode = (SValueNode *)cell->pNode;
SFltSclDatum valDatum;
FLT_ERR_RET(fltSclBuildDatumFromValueNode(&valDatum, valueNode));
if(valueNode->node.resType.type == TSDB_DATA_TYPE_FLOAT || valueNode->node.resType.type == TSDB_DATA_TYPE_DOUBLE) {
minDatum.i = TMIN(minDatum.i, valDatum.d);
maxDatum.i = TMAX(maxDatum.i, valDatum.d);
} else {
minDatum.i = TMIN(minDatum.i, valDatum.i);
maxDatum.i = TMAX(maxDatum.i, valDatum.i);
}
cell = cell->pNext;
}
SFltSclPoint startPt = {.start = true, .excl = false, .val = minDatum};
SFltSclPoint endPt = {.start = false, .excl = false, .val = maxDatum};
if (NULL == taosArrayPush(points, &startPt)) {
FLT_ERR_RET(terrno);
}
if (NULL == taosArrayPush(points, &endPt)) {
FLT_ERR_RET(terrno);
}
break;
}
default: { default: {
qError("not supported operator type : %d when build range points", oper->type); qError("not supported operator type : %d when build range points", oper->type);
break; break;
@ -5075,11 +5105,13 @@ static bool fltSclIsCollectableNode(SNode *pNode) {
if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL || if (!(pOper->opType == OP_TYPE_GREATER_THAN || pOper->opType == OP_TYPE_GREATER_EQUAL ||
pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL || pOper->opType == OP_TYPE_LOWER_THAN || pOper->opType == OP_TYPE_LOWER_EQUAL ||
pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL)) { pOper->opType == OP_TYPE_NOT_EQUAL || pOper->opType == OP_TYPE_EQUAL ||
pOper->opType == OP_TYPE_IN)) {
return false; return false;
} }
if (!(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE)) { if (!((nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_VALUE) ||
(nodeType(pOper->pLeft) == QUERY_NODE_COLUMN && nodeType(pOper->pRight) == QUERY_NODE_NODE_LIST))) {
return false; return false;
} }
return true; return true;

2
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -3364,7 +3364,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) {
return 0; return 0;
} }
void streamStateCurNext_rocksdb(SStreamStateCur* pCur) { void streamStateCurNext_rocksdb(SStreamStateCur* pCur) {
if (pCur) { if (pCur && pCur->iter && rocksdb_iter_valid(pCur->iter)) {
rocksdb_iter_next(pCur->iter); rocksdb_iter_next(pCur->iter);
} }
} }

15
source/libs/sync/src/syncAppendEntries.c
View File

@ -95,6 +95,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
bool accepted = false; bool accepted = false;
SSyncRaftEntry* pEntry = NULL; SSyncRaftEntry* pEntry = NULL;
bool resetElect = false; bool resetElect = false;
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@ -150,10 +152,10 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
goto _IGNORE; goto _IGNORE;
} }
sTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64 sGTrace("vgId:%d, recv append entries msg. index:%" PRId64 ", term:%" PRId64 ", preLogIndex:%" PRId64
", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 " entryterm:%" PRId64, ", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 " entryterm:%" PRId64,
pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex,
pEntry->term); pEntry->term);
if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) { if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) {
pReply->fsmState = ths->fsmState; pReply->fsmState = ths->fsmState;
@ -179,6 +181,11 @@ _SEND_RESPONSE:
sTrace("vgId:%d, update commit return index %" PRId64 "", ths->vgId, returnIndex); sTrace("vgId:%d, update commit return index %" PRId64 "", ths->vgId, returnIndex);
} }
TRACE_SET_MSGID(&(rpcRsp.info.traceId), tGenIdPI64());
trace = &(rpcRsp.info.traceId);
sGTrace("vgId:%d, send append reply matchIndex:%" PRId64 " term:%" PRId64 " lastSendIndex:%" PRId64
" to dest: 0x%016" PRIx64,
ths->vgId, pReply->matchIndex, pReply->term, pReply->lastSendIndex, pReply->destId.addr);
// ack, i.e. send response // ack, i.e. send response
TAOS_CHECK_RETURN(syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp)); TAOS_CHECK_RETURN(syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp));

6
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -43,6 +43,8 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
int32_t code = 0; int32_t code = 0;
SyncAppendEntriesReply* pMsg = (SyncAppendEntriesReply*)pRpcMsg->pCont; SyncAppendEntriesReply* pMsg = (SyncAppendEntriesReply*)pRpcMsg->pCont;
int32_t ret = 0; int32_t ret = 0;
const STraceId* trace = &pRpcMsg->info.traceId;
char tbuf[40] = {0};
// if already drop replica, do not process // if already drop replica, do not process
if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
@ -63,8 +65,8 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
return TSDB_CODE_SYN_WRONG_TERM; return TSDB_CODE_SYN_WRONG_TERM;
} }
sTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "", sGTrace("vgId:%d, received append entries reply. srcId:0x%016" PRIx64 ", term:%" PRId64 ", matchIndex:%" PRId64 "",
pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex); pMsg->vgId, pMsg->srcId.addr, pMsg->term, pMsg->matchIndex);
if (pMsg->success) { if (pMsg->success) {
SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId));

24
source/libs/sync/src/syncPipeline.c
View File

@ -1026,6 +1026,14 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn
int32_t code = 0; int32_t code = 0;
if (pMgr->restored != false) return TSDB_CODE_SYN_INTERNAL_ERROR; if (pMgr->restored != false) return TSDB_CODE_SYN_INTERNAL_ERROR;
sTrace("vgId:%d, begin to recover sync log repl. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 ", %" PRId64
", %" PRId64 ") restore:%d, buffer: [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64
"), msg: {lastSendIndex:%" PRId64 ", matchIndex:%" PRId64 ", fsmState:%d, success:%d, lastMatchTerm:%" PRId64
"}",
pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pMgr->restored,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex, pMsg->lastSendIndex, pMsg->matchIndex,
pMsg->fsmState, pMsg->success, pMsg->lastMatchTerm);
if (pMgr->endIndex == 0) { if (pMgr->endIndex == 0) {
if (pMgr->startIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR; if (pMgr->startIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
if (pMgr->matchIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR; if (pMgr->matchIndex != 0) return TSDB_CODE_SYN_INTERNAL_ERROR;
@ -1171,6 +1179,11 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
int64_t nowMs = taosGetMonoTimestampMs(); int64_t nowMs = taosGetMonoTimestampMs();
int32_t code = 0; int32_t code = 0;
sTrace("vgId:%d, begin to probe peer:%" PRIx64 " with msg of index:%" PRId64 ". repl-mgr:[%" PRId64 ", %" PRId64
", %" PRId64 "), restored:%d",
pNode->vgId, pNode->replicasId[pMgr->peerId].addr, index, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pMgr->restored);
if (pMgr->endIndex > pMgr->startIndex && if (pMgr->endIndex > pMgr->startIndex &&
nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) { nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) {
return 0; return 0;
@ -1206,6 +1219,10 @@ int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex inde
int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
if (!pMgr->restored) return TSDB_CODE_SYN_INTERNAL_ERROR; if (!pMgr->restored) return TSDB_CODE_SYN_INTERNAL_ERROR;
sTrace("vgId:%d, begin to attempt replicate log entries from end to match. repl-mgr:[%" PRId64 ", %" PRId64
", %" PRId64 "), restore:%d",
pNode->vgId, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pMgr->restored);
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff));
int32_t code = 0; int32_t code = 0;
@ -1527,11 +1544,12 @@ int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex ind
goto _err; goto _err;
} }
TRACE_SET_MSGID(&(msgOut.info.traceId), tGenIdPI64());
STraceId* trace = &(msgOut.info.traceId);
sGTrace("vgId:%d, replicate one msg index:%" PRId64 " term:%" PRId64 " prevterm:%" PRId64 " to dest: 0x%016" PRIx64,
pNode->vgId, pEntry->index, pEntry->term, prevLogTerm, pDestId->addr);
TAOS_CHECK_GOTO(syncNodeSendAppendEntries(pNode, pDestId, &msgOut), &lino, _err); TAOS_CHECK_GOTO(syncNodeSendAppendEntries(pNode, pDestId, &msgOut), &lino, _err);
sTrace("vgId:%d, replicate one msg index:%" PRId64 " term:%" PRId64 " prevterm:%" PRId64 " to dest: 0x%016" PRIx64,
pNode->vgId, pEntry->index, pEntry->term, prevLogTerm, pDestId->addr);
if (!inBuf) { if (!inBuf) {
syncEntryDestroy(pEntry); syncEntryDestroy(pEntry);
pEntry = NULL; pEntry = NULL;

4
source/libs/sync/src/syncUtil.c
View File

@ -152,8 +152,8 @@ static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLe
for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { for (int32_t i = 0; i < pSyncNode->replicaNum; i++) {
SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i]; SSyncLogReplMgr* pMgr = pSyncNode->logReplMgrs[i];
if (pMgr == NULL) break; if (pMgr == NULL) break;
len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 " %" PRId64 ", %" PRId64 "]", i, pMgr->restored, len += tsnprintf(buf + len, bufLen - len, "%d:%d [%" PRId64 ", %" PRId64 ", %" PRId64 "]", i, pMgr->restored,
pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex); pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex);
if (i + 1 < pSyncNode->replicaNum) { if (i + 1 < pSyncNode->replicaNum) {
len += tsnprintf(buf + len, bufLen - len, "%s", ", "); len += tsnprintf(buf + len, bufLen - len, "%s", ", ");
} }

2
source/libs/transport/src/transCli.c
View File

@ -3090,7 +3090,7 @@ int32_t transReleaseCliHandle(void* handle) {
static int32_t transInitMsg(void* pInstRef, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx, SCliReq** pCliMsg) { static int32_t transInitMsg(void* pInstRef, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx, SCliReq** pCliMsg) {
int32_t code = 0; int32_t code = 0;
TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64());
SCliReq* pCliReq = NULL; SCliReq* pCliReq = NULL;
SReqCtx* pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); SReqCtx* pCtx = taosMemoryCalloc(1, sizeof(SReqCtx));

2
source/util/src/tlog.c
View File

@ -128,7 +128,7 @@ int32_t idxDebugFlag = 131;
int32_t sndDebugFlag = 131; int32_t sndDebugFlag = 131;
int32_t simDebugFlag = 131; int32_t simDebugFlag = 131;
int32_t tqClientDebug = 0; int32_t tqClientDebugFlag = 131;
int64_t dbgEmptyW = 0; int64_t dbgEmptyW = 0;
int64_t dbgWN = 0; int64_t dbgWN = 0;

9
tests/army/frame/common.py
View File

@ -803,11 +803,14 @@ class TDCom:
else: else:
tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}") tdLog.exit(f"getOneRow out of range: row_index={location} row_count={self.query_row}")
def killProcessor(self, processorName): def kill_signal_process(self, signal=15, processor_name: str = "taosd"):
if (platform.system().lower() == 'windows'): if (platform.system().lower() == 'windows'):
os.system("TASKKILL /F /IM %s.exe"%processorName) os.system(f"TASKKILL /F /IM {processor_name}.exe")
else: else:
os.system("unset LD_PRELOAD; pkill %s " % processorName) command = f"unset LD_PRELOAD; sudo pkill -f -{signal} '{processor_name}'"
tdLog.debug(f"command: {command}")
os.system(command)
def gen_tag_col_str(self, gen_type, data_type, count): def gen_tag_col_str(self, gen_type, data_type, count):
""" """

2
tests/army/frame/sql.py
View File

@ -73,7 +73,7 @@ class TDSql:
for k, v in kwargs.items(): for k, v in kwargs.items():
s += f" {k} {v}" s += f" {k} {v}"
if "duration" not in kwargs: if "duration" not in kwargs:
s += " duration 300" s += " duration 100"
self.cursor.execute(s) self.cursor.execute(s)
s = f'use {dbname}' s = f'use {dbname}'
self.cursor.execute(s) self.cursor.execute(s)

2
tests/army/storage/s3/s3Basic.json
View File

@ -20,7 +20,7 @@
"replica": 1, "replica": 1,
"duration":"10d", "duration":"10d",
"s3_keeplocal":"30d", "s3_keeplocal":"30d",
"s3_chunksize":"131072", "s3_chunkpages":"131072",
"tsdb_pagesize":"1", "tsdb_pagesize":"1",
"s3_compact":"1", "s3_compact":"1",
"wal_retention_size":"1", "wal_retention_size":"1",

10
tests/army/storage/s3/s3Basic.py
View File

@ -168,13 +168,13 @@ class TDTestCase(TBase):
if keepLocal is not None: if keepLocal is not None:
kw1 = f"s3_keeplocal {keepLocal}" kw1 = f"s3_keeplocal {keepLocal}"
if chunkSize is not None: if chunkSize is not None:
kw2 = f"s3_chunksize {chunkSize}" kw2 = f"s3_chunkpages {chunkSize}"
if compact is not None: if compact is not None:
kw3 = f"s3_compact {compact}" kw3 = f"s3_compact {compact}"
sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}" sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}"
tdSql.execute(sql, show=True) tdSql.execute(sql, show=True)
#sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';" #sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';"
sql = f"select * from information_schema.ins_databases where name='db1';" sql = f"select * from information_schema.ins_databases where name='db1';"
tdSql.query(sql) tdSql.query(sql)
# 29 30 31 -> chunksize keeplocal compact # 29 30 31 -> chunksize keeplocal compact
@ -194,9 +194,9 @@ class TDTestCase(TBase):
f"create database db2 s3_keeplocal -1", f"create database db2 s3_keeplocal -1",
f"create database db2 s3_keeplocal 0", f"create database db2 s3_keeplocal 0",
f"create database db2 s3_keeplocal 365001", f"create database db2 s3_keeplocal 365001",
f"create database db2 s3_chunksize -1", f"create database db2 s3_chunkpages -1",
f"create database db2 s3_chunksize 0", f"create database db2 s3_chunkpages 0",
f"create database db2 s3_chunksize 900000000", f"create database db2 s3_chunkpages 900000000",
f"create database db2 s3_compact -1", f"create database db2 s3_compact -1",
f"create database db2 s3_compact 100", f"create database db2 s3_compact 100",
f"create database db2 duration 1d s3_keeplocal 1d" f"create database db2 duration 1d s3_keeplocal 1d"

2
tests/army/storage/s3/s3Basic1.json
View File

@ -20,7 +20,7 @@
"replica": 1, "replica": 1,
"duration":"10d", "duration":"10d",
"s3_keeplocal":"30d", "s3_keeplocal":"30d",
"s3_chunksize":"131072", "s3_chunkpages":"131072",
"tsdb_pagesize":"1", "tsdb_pagesize":"1",
"s3_compact":"1", "s3_compact":"1",
"wal_retention_size":"1", "wal_retention_size":"1",

10
tests/army/storage/s3/s3azure.py
View File

@ -202,13 +202,13 @@ class TDTestCase(TBase):
if keepLocal is not None: if keepLocal is not None:
kw1 = f"s3_keeplocal {keepLocal}" kw1 = f"s3_keeplocal {keepLocal}"
if chunkSize is not None: if chunkSize is not None:
kw2 = f"s3_chunksize {chunkSize}" kw2 = f"s3_chunkpages {chunkSize}"
if compact is not None: if compact is not None:
kw3 = f"s3_compact {compact}" kw3 = f"s3_compact {compact}"
sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}" sql = f" create database db1 vgroups 1 duration 1h {kw1} {kw2} {kw3}"
tdSql.execute(sql, show=True) tdSql.execute(sql, show=True)
# sql = f"select name,s3_keeplocal,s3_chunksize,s3_compact from information_schema.ins_databases where name='db1';" # sql = f"select name,s3_keeplocal,s3_chunkpages,s3_compact from information_schema.ins_databases where name='db1';"
sql = f"select * from information_schema.ins_databases where name='db1';" sql = f"select * from information_schema.ins_databases where name='db1';"
tdSql.query(sql) tdSql.query(sql)
# 29 30 31 -> chunksize keeplocal compact # 29 30 31 -> chunksize keeplocal compact
@ -228,9 +228,9 @@ class TDTestCase(TBase):
f"create database db2 s3_keeplocal -1", f"create database db2 s3_keeplocal -1",
f"create database db2 s3_keeplocal 0", f"create database db2 s3_keeplocal 0",
f"create database db2 s3_keeplocal 365001", f"create database db2 s3_keeplocal 365001",
f"create database db2 s3_chunksize -1", f"create database db2 s3_chunkpages -1",
f"create database db2 s3_chunksize 0", f"create database db2 s3_chunkpages 0",
f"create database db2 s3_chunksize 900000000", f"create database db2 s3_chunkpages 900000000",
f"create database db2 s3_compact -1", f"create database db2 s3_compact -1",
f"create database db2 s3_compact 100", f"create database db2 s3_compact 100",
f"create database db2 duration 1d s3_keeplocal 1d" f"create database db2 duration 1d s3_keeplocal 1d"

337
tests/army/tmq/drop_lost_comsumers.py Normal file
View File

@ -0,0 +1,337 @@
import time
import os
import threading
import datetime
from taos.tmq import Consumer
from taos.error import TmqError
from frame.log import tdLog
from frame.cases import tdCases
from frame.sql import tdSql
from frame.caseBase import *
from frame import etool
from frame.common import tdCom
class TaosConsumer:
# TODO: Move this class to tq.py and remove it from here
def __init__(self):
self.sub_once = True
self.once_consumer_rows = 0
self.sub_log = False
self.safe_counter = ThreadSafeCounter()
def log_info(self, message):
if self.sub_log:
tdLog.info(message)
#TODO merge sub_consumer and sub_consumer_once
def sub_consumer(self, consumer, group_id, topic_name):
group_id = int(group_id)
if group_id < 100:
try:
consumer.subscribe([topic_name])
except TmqError:
tdLog.exit(f"subscribe error")
nrows = 0
while True:
start = datetime.datetime.now()
tdLog.info(f"time:{start},consumer:{group_id}, start to consume")
message = consumer.poll(timeout=10.0)
if message:
message_offset = message.offset()
# topic = message.topic()
# database = message.database()
for block in message:
addrows = block.nrows()
nrows += block.nrows()
ncols = block.ncols()
# values = block.fetchall
end = datetime.datetime.now()
elapsed_time = end - start
tdLog.info(
f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},"
f"consumer_nrows:{nrows},consumer_addrows:{addrows},"
f"consumer_ncols:{ncols},offset:{id}"
)
consumer.commit()
tdLog.info(f"consumer:{group_id},consumer_nrows:{nrows}")
# consumer.unsubscribe()
# consumer.close()
def set_conf(
self,
td_connect_ip="localhost",
group_id=1,
client_id="test_consumer_py",
enable_auto_commit="false",
auto_commit_interval_ms="1000",
auto_offset_reset="earliest",
msg_with_table_name="true",
session_timeout_ms=10000,
max_poll_interval_ms=180000,
experimental_snapshot_enable="false",
):
conf = {
# auth options
# consume options
"td.connect.ip": f"{td_connect_ip}",
"group.id": f"{group_id}",
"client.id": f"{client_id}",
"enable.auto.commit": f"{enable_auto_commit}",
"auto.commit.interval.ms": f"{auto_commit_interval_ms}",
"auto.offset.reset": f"{auto_offset_reset}",
"msg.with.table.name": f"{msg_with_table_name}",
"session.timeout.ms": f"{session_timeout_ms}",
"max.poll.interval.ms": f"{max_poll_interval_ms}",
"experimental.snapshot.enable": f"{experimental_snapshot_enable}",
}
return conf
def sub_consumer_once(self, consumer, group_id, topic_name, stop_event):
group_id = int(group_id)
if group_id < 100:
consumer.subscribe([topic_name])
nrows = 0
consumer_nrows = 0
count = 0
while not stop_event.is_set():
start = datetime.datetime.now()
# self.log_info(
# f"time:{start},consumer:{group_id}, start to consume,consumer_nrows:{consumer_nrows}"
# )
message = None
if consumer_nrows < self.once_consumer_rows:
message = consumer.poll(timeout=1.0)
elif consumer_nrows >= self.once_consumer_rows:
if count == 0:
# when break the loop, the consumer will be closed, so we need to continue to keep consumer alive util the stop_event is set
tdLog.info("stop consumer when consumer all rows")
count += 1
# tdLog.info("stop consumer when consumer all rows")
else:
continue
if message:
message_offset = message.offset()
# topic = message.topic()
# database = message.database()
for block in message:
addrows = block.nrows()
nrows += block.nrows()
self.safe_counter.rows(block.nrows())
ncols = block.ncols()
# values = block.fetchall
end = datetime.datetime.now()
elapsed_time = end - start
# self.log_info(
# f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},consumer_nrows:{nrows},consumer_addrows:{addrows}, consumer_ncols:{ncols},offset:{message_offset}"
# )
self.log_info(
f"consumer:{group_id},consumer_nrows:{nrows},counter.counter:{self.safe_counter.counter},counter.get():{self.safe_counter.get()}"
)
# consumer.commit()
consumer_nrows = nrows
tdLog.info("Consumer subscription thread is stopping.")
def taosc_consumer(self, conf: list, topic_name: str, stop_event: threading.Event):
try:
tdLog.info(conf)
tdLog.info("start to config consumer")
consumer = Consumer(conf)
tdLog.info("start to subscribe")
group_id = int(conf["group.id"])
tdLog.info(f"{consumer},{group_id}")
if self.sub_once:
self.sub_consumer_once(consumer, group_id, topic_name, stop_event)
else:
self.sub_consumer(consumer, group_id, topic_name)
# only consumer once
except Exception as e:
tdLog.exit(f"{e}")
# consumer.close()
class ThreadSafeCounter:
def __init__(self):
self.counter = 0
self.lock = threading.Lock()
def rows(self, rows):
with self.lock:
self.counter += rows
def get(self):
with self.lock:
return self.counter
class TDTestCase:
# updatecfgDict = {'debugFlag': 135, 'asynclog': 0}
def __init__(self):
# db parameter
self.table_number = 1000
self.rows_per_table = 1000
# consumer parameter
self.consumer_groups_num = 2
self.session_timeout_ms = 180000
self.max_poll_interval_ms = 180000
# case consumer parameter
self.consumer_rows_per_thread = self.table_number * self.rows_per_table
self.consumer_all_rows = (
self.consumer_rows_per_thread * self.consumer_groups_num
)
self.topic_name = "select_d1"
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), logSql)
self.consumer_instance = TaosConsumer()
# tdSql.init(conn.cursor(), logSql) # output sql.txt file
def caseDescription(self):
"""
drop_lost_consmuers<hrchen>:
1. verifying that the boundary and valid values of session_timeout_ms are in effect
2. verifying that the boundary and valid values of max_poll_interval_ms are in effect
3. verifying that consumer will be closed when the session_timeout_ms and max_poll_interval_ms is expired
"""
return
def check_consumer(self, count, rows, stop_event=None):
time.sleep(count)
try:
tdLog.info(
f"wait timeout count:{count} and check consumer status whether is closed"
)
for _ in range(2):
tdSql.query("show consumers")
anser_rows = tdSql.getRows()
if anser_rows == rows:
break
time.sleep(1)
tdLog.info(
f"wait for {count} seconds to check that consumers number is {anser_rows}"
)
if anser_rows != rows:
if stop_event:
stop_event.set()
tdLog.exit(f"consumer number is {anser_rows } but not expected {rows}")
except Exception as e:
tdLog.exit(f"{e},check consumer error")
def drop_session_timeout_consmuers(self):
tdSql.execute(f"drop topic if exists {self.topic_name};")
tdSql.execute("use db_sub")
tdSql.execute(f"create topic {self.topic_name} as select * from db_sub.meters;")
# start consumer and config some parameters
os.system(
f"nohup python3 ./tmq/per_consumer.py -c {self.consumer_groups_num} -s {self.session_timeout_ms} -p {self.max_poll_interval_ms} -t {self.topic_name} > consumer.log &"
)
# wait 5s for consuming data
time.sleep(5)
# kill consumer to simulate session_timeout_ms
tdLog.info("kill per_consumer.py")
tdCom.kill_signal_process(
signal=9, processor_name=r"python3\s*./tmq/per_consumer.py"
)
self.check_consumer(int(self.session_timeout_ms / 1000), 0)
tdSql.execute(f"drop topic if exists {self.topic_name};")
os.system("rm -rf consumer.log")
def drop_max_poll_timeout_consmuers(self):
tdSql.execute(f"drop topic if exists {self.topic_name};")
tdSql.execute("use db_sub")
tdSql.execute(f"create topic {self.topic_name} as select * from db_sub.meters;")
threads = []
self.safe_counter = ThreadSafeCounter()
self.consumer_instance.safe_counter = self.safe_counter
stop_event = threading.Event()
self.consumer_instance.once_consumer_rows = self.consumer_rows_per_thread
tdLog.info(f"consumer_rows:{self.consumer_instance.once_consumer_rows}")
self.consumer_instance.sub_once = True
for group_id in range(self.consumer_groups_num):
conf = self.consumer_instance.set_conf(
group_id=group_id,
session_timeout_ms=self.session_timeout_ms,
max_poll_interval_ms=self.max_poll_interval_ms,
)
threads.append(
threading.Thread(
target=self.consumer_instance.taosc_consumer,
args=(conf, self.topic_name, stop_event),
)
)
for tr in threads:
tr.start()
while True:
if self.safe_counter.counter < self.consumer_all_rows:
# control print log frequency
time.sleep(1)
tdLog.info(
f"consumer_all_rows:{self.consumer_all_rows},counter.get():{self.safe_counter.counter}"
)
elif self.safe_counter.counter == self.consumer_all_rows:
# adding 5s is for heartbeat check
self.check_consumer(int(self.max_poll_interval_ms / 1000 ) + 5, 0, stop_event)
stop_event.set()
break
time.sleep(1)
tdSql.execute(f"drop topic if exists {self.topic_name};")
def case_session_timeout(self):
"""
TEST CASE: verifying that the boundary and valid values of session_timeout_ms are in effect
"""
tdLog.info("start to test session_timeout_ms=12s")
# test session_timeout_ms=12s
self.session_timeout_ms = 12000
self.max_poll_interval_ms = 180000
# self.set_session_timeout = int(self.session_timeout_ms / 1000)
self.drop_session_timeout_consmuers()
tdLog.info("stop to test session_timeout_ms=12s and done ")
def case_max_poll_timeout(self):
"""
TEST CASE: verifying that the boundary and valid values of max_poll_interval_ms are in effect
"""
tdLog.info("start to test max_poll_interval_ms=20s")
# test max_poll_interval_ms=20s
self.session_timeout_ms = 300000
self.max_poll_interval_ms = 20000
self.drop_max_poll_timeout_consmuers()
tdLog.info("stop to test max_poll_interval_ms=20s and done ")
def run(self):
"""
Run the test cases for session timeout and max poll timeout.
"""
vgroups = 4
etool.benchMark(
command=f"-d db_sub -t {self.table_number} -n {self.rows_per_table} -v {vgroups} -a {self.replicaVar} -y"
)
# test case start here
self.topic_name = "select_d1"
# self.case_session_timeout()
self.case_max_poll_timeout()
def stop(self):
"""
Closes the taos connection and logs the success message.
"""
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

182
tests/army/tmq/per_consumer.py Normal file
View File

@ -0,0 +1,182 @@
import os
import taos
import sys
from datetime import datetime
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from frame.log import tdLog
import subprocess
from multiprocessing import Process
import threading
from taos.tmq import Consumer
import click
# TDDO
# 1. using tmq common class to replace the function, file drop_lost_consumers.py has the same function
try:
conn = taos.connect()
except Exception as e:
tdLog.info(str(e))
@click.command()
@click.option(
"-c",
"--consumer-groups-num",
"consumer_group_num",
default=1,
help="Number of consumer group.",
)
@click.option(
"-s",
"--session-timeout-ms",
"session_timeout_ms",
default=60000,
help="session timeout:ms",
)
@click.option(
"-p",
"--max-poll-interval-ms",
"max_poll_interval_ms",
default=180000,
help="max poll interval timeout:ms",
)
@click.option(
"-t",
"--topic-name",
"topic_name",
default="select_d1",
help="topic name",
)
def test_timeout_sub(consumer_group_num, session_timeout_ms, max_poll_interval_ms, topic_name):
threads = []
tdLog.info(f"consumer_group_num:{consumer_group_num}, session_timeout_ms:{session_timeout_ms}, max_poll_interval_ms:{max_poll_interval_ms}")
for id in range(consumer_group_num):
conf = set_conf(
group_id=id,
session_timeout_ms=session_timeout_ms,
max_poll_interval_ms=max_poll_interval_ms,
)
tdLog.info(f"conf:{conf}")
threads.append(threading.Thread(target=taosc_consumer, args=(conf,topic_name)))
for tr in threads:
tr.start()
for tr in threads:
tr.join()
def sub_consumer(consumer, group_id, topic_name):
group_id = int(group_id)
if group_id < 100:
try:
consumer.subscribe([topic_name])
except Exception as e:
tdLog.info(f"subscribe error")
exit(1)
nrows = 0
while True:
start = datetime.now()
tdLog.info(f"time:{start},consumer:{group_id}, start to consume")
message = consumer.poll(timeout=10.0)
if message:
id = message.offset()
topic = message.topic()
database = message.database()
for block in message:
addrows = block.nrows()
nrows += block.nrows()
ncols = block.ncols()
values = block.fetchall
end = datetime.now()
elapsed_time = end - start
tdLog.info(
f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},consumer_nrows:{nrows},consumer_addrows:{addrows}, consumer_ncols:{ncols},offset:{id}"
)
consumer.commit()
tdLog.info(f"consumer:{group_id},consumer_nrows:{nrows}")
# consumer.unsubscribe()
# consumer.close()
def sub_consumer_once(consumer, group_id, topic_name):
group_id = int(group_id)
if group_id < 100:
consumer.subscribe([topic_name])
nrows = 0
consumer_nrows = 0
while True:
start = datetime.now()
tdLog.info(f"time:{start},consumer:{group_id}, start to consume")
# start = datetime.now()
# tdLog.info(f"time:{start},consumer:{group_id}, start to consume")
tdLog.info(f"consumer_nrows:{consumer_nrows}")
if consumer_nrows < 1000000:
message = consumer.poll(timeout=10.0)
else:
tdLog.info(" stop consumer when consumer all rows")
if message:
id = message.offset()
topic = message.topic()
database = message.database()
for block in message:
addrows = block.nrows()
nrows += block.nrows()
ncols = block.ncols()
values = block.fetchall
end = datetime.now()
elapsed_time = end - start
# tdLog.info(f"time:{end},consumer:{group_id}, elapsed time:{elapsed_time},consumer_nrows:{nrows},consumer_addrows:{addrows}, consumer_ncols:{ncols},offset:{id}")
consumer.commit()
# tdLog.info(f"consumer:{group_id},consumer_nrows:{nrows}")
consumer_nrows = nrows
# consumer.unsubscribe()
# consumer.close()
# break
def set_conf(
td_connect_ip="localhost",
group_id=1,
client_id="test_consumer_py",
enable_auto_commit="false",
auto_commit_interval_ms="1000",
auto_offset_reset="earliest",
msg_with_table_name="true",
session_timeout_ms=10000,
max_poll_interval_ms=20000,
experimental_snapshot_enable="false",
):
conf = {
# auth options
# consume options
"td.connect.ip": f"{td_connect_ip}",
"group.id": f"{group_id}",
"client.id": f"{client_id}",
"enable.auto.commit": f"{enable_auto_commit}",
"auto.commit.interval.ms": f"{auto_commit_interval_ms}",
"auto.offset.reset": f"{auto_offset_reset}",
"msg.with.table.name": f"{msg_with_table_name}",
"session.timeout.ms": f"{session_timeout_ms}",
"max.poll.interval.ms": f"{max_poll_interval_ms}",
"experimental.snapshot.enable": f"{experimental_snapshot_enable}",
}
return conf
def taosc_consumer(conf,topic_name):
consumer = Consumer(conf)
group_id = int(conf["group.id"])
tdLog.info(f"{consumer},{group_id}")
try:
sub_consumer_once(consumer, group_id, topic_name)
except Exception as e:
tdLog.info(str(e))
if __name__ == "__main__":
test_timeout_sub()

12
tests/develop-test/2-query/show_create_db.py
View File

@ -42,17 +42,17 @@ class TDTestCase:
tdSql.query('show create database scd;') tdSql.query('show create database scd;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd') tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
tdSql.query('show create database scd2;') tdSql.query('show create database scd2;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2') tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
tdSql.query('show create database scd4') tdSql.query('show create database scd4')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4') tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
self.restartTaosd(1, dbname='scd') self.restartTaosd(1, dbname='scd')
@ -60,16 +60,16 @@ class TDTestCase:
tdSql.query('show create database scd;') tdSql.query('show create database scd;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd') tdSql.checkData(0, 0, 'scd')
tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 2 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
tdSql.query('show create database scd2;') tdSql.query('show create database scd2;')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd2') tdSql.checkData(0, 0, 'scd2')
tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd2` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 3 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
tdSql.query('show create database scd4') tdSql.query('show create database scd4')
tdSql.checkRows(1) tdSql.checkRows(1)
tdSql.checkData(0, 0, 'scd4') tdSql.checkData(0, 0, 'scd4')
tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKSIZE 262144 S3_KEEPLOCAL 5256000m S3_COMPACT 0") tdSql.checkData(0, 1, "CREATE DATABASE `scd4` BUFFER 256 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 10d WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 STT_TRIGGER 13 KEEP 3650d,3650d,3650d PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0 TABLE_PREFIX 0 TABLE_SUFFIX 0 TSDB_PAGESIZE 4 WAL_RETENTION_PERIOD 3600 WAL_RETENTION_SIZE 0 KEEP_TIME_OFFSET 0 ENCRYPT_ALGORITHM 'none' S3_CHUNKPAGES 131072 S3_KEEPLOCAL 525600m S3_COMPACT 1")
tdSql.execute('drop database scd') tdSql.execute('drop database scd')

2
tests/parallel_test/cases.task
View File

@ -47,7 +47,7 @@
,,y,army,./pytest.sh python3 ./test.py -f query/window/base.py ,,y,army,./pytest.sh python3 ./test.py -f query/window/base.py
,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3 ,,y,army,./pytest.sh python3 ./test.py -f query/sys/tb_perf_queries_exist_test.py -N 3
,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py ,,y,army,./pytest.sh python3 ./test.py -f query/test_having.py
,,n,army,python3 ./test.py -f tmq/drop_lost_comsumers.py
# #
# system test # system test
# #

2
tests/pytest/util/sql.py
View File

@ -104,7 +104,7 @@ class TDSql:
for k, v in kwargs.items(): for k, v in kwargs.items():
s += f" {k} {v}" s += f" {k} {v}"
if "duration" not in kwargs: if "duration" not in kwargs:
s += " duration 300" s += " duration 100"
self.cursor.execute(s) self.cursor.execute(s)
s = f'use {dbname}' s = f'use {dbname}'
self.cursor.execute(s) self.cursor.execute(s)

6
tests/script/tsim/db/create_all_options.sim
View File

@ -128,6 +128,12 @@ endi
if $data21_db != 3000 then # wal_fsync_period if $data21_db != 3000 then # wal_fsync_period
return -1 return -1
endi endi
if $data30_db != 525600m then # s3_keeplocal
return -1
endi
if $data31_db != 1 then # s3_compact
return -1
endi
sql drop database db sql drop database db

2
tests/script/tsim/insert/commit-merge0.sim
View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sql connect sql connect
print =============== create database print =============== create database
sql create database db duration 300 keep 365000d,365000d,365000d sql create database db duration 120 keep 365000d,365000d,365000d
sql select * from information_schema.ins_databases sql select * from information_schema.ins_databases
if $rows != 3 then if $rows != 3 then
return -1 return -1

2
tests/script/tsim/query/interval-offset.sim
View File

@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
sql connect sql connect
print =============== create database print =============== create database
sql create database d0 duration 300 sql create database d0 duration 120
sql use d0 sql use d0
print =============== create super table and child table print =============== create super table and child table

49
tests/script/tsim/scalar/in.sim
View File

@ -35,6 +35,14 @@ if $rows != 3 then
return -1 return -1
endi endi
sql explain verbose true select * from tb1 where tts in ('2022-07-10 16:31:01', '2022-07-10 16:31:03', 1657441865000);
if $rows != 3 then
return -1
endi
if $data20 != @ Time Range: [-9223372036854775808, 9223372036854775807]@ then
return -1
endi
sql select * from tb1 where fbool in (0, 3); sql select * from tb1 where fbool in (0, 3);
if $rows != 5 then if $rows != 5 then
return -1 return -1
@ -80,4 +88,45 @@ if $rows != 0 then
return -1 return -1
endi endi
sql explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000);
if $rows != 4 then
return -1
endi
if $data20 != @ Time Range: [1657441840000, 1657441980000]@ then
return -1
endi
sql explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000, true);
if $rows != 4 then
return -1
endi
if $data20 != @ Time Range: [1, 1657441980000]@ then
return -1
endi
sql explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000, false);
if $rows != 4 then
return -1
endi
if $data20 != @ Time Range: [0, 1657441980000]@ then
return -1
endi
sql explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000, 1.02);
if $rows != 4 then
return -1
endi
if $data20 != @ Time Range: [1, 1657441980000]@ then
return -1
endi
sql explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000, -1.02);
if $rows != 4 then
return -1
endi
if $data20 != @ Time Range: [-1, 1657441980000]@ then
return -1
endi
sql_error explain verbose true select * from tb1 where fts in ('2022-07-10 16:31:00', '2022-07-10 16:33:00', 1657441840000, 'abc');
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

8
tests/script/tsim/sma/drop_sma.sim
View File

@ -126,21 +126,21 @@ sql drop table stb;
print ========== step5 print ========== step5
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
print ========== step6 repeat print ========== step6 repeat
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int);
sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
print ========== step7 print ========== step7
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
@ -160,7 +160,7 @@ sql DROP INDEX sma_index_3 ;
print ========== step8 print ========== step8
sql drop database if exists db; sql drop database if exists db;
sleep 2000 sleep 2000
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);

8
tests/script/tsim/sma/sma_leak.sim
View File

@ -98,21 +98,21 @@ sql drop table stb;
print ========== step5 print ========== step5
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint, c_float float, c_double double, c_bool bool, c_binary binary(16), c_nchar nchar(32), c_ts timestamp, c_tint_un tinyint unsigned, c_sint_un smallint unsigned, c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
print ========== step6 repeat print ========== step6 repeat
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint ) tags (t_int int);
sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s; sql CREATE SMA INDEX sma_index_1 ON stb1 function(min(c_int), max(c_int)) interval(6m, 10s) sliding(6m) watermark 5s;
print ========== step7 print ========== step7
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);
@ -131,7 +131,7 @@ sql DROP INDEX sma_index_3 ;
print ========== step8 print ========== step8
sql drop database if exists db; sql drop database if exists db;
sql create database db duration 300; sql create database db duration 120;
sql use db; sql use db;
sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int); sql create table stb1(ts timestamp, c_int int, c_bint bigint, c_sint smallint, c_tint tinyint,c_float float, c_double double, c_bool bool,c_binary binary(16), c_nchar nchar(32), c_ts timestamp,c_tint_un tinyint unsigned, c_sint_un smallint unsigned,c_int_un int unsigned, c_bint_un bigint unsigned) tags (t_int int);

2
tests/system-test/0-others/compatibility.py
View File

@ -32,7 +32,7 @@ class TDTestCase:
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor()) tdSql.init(conn.cursor())
self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 100 stt_trigger 1; ;use deldata;
create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int);
create table deldata.ct1 using deldata.stb1 tags ( 1 ); create table deldata.ct1 using deldata.stb1 tags ( 1 );
insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );

2
tests/system-test/0-others/compatibility_coverage.py
View File

@ -30,7 +30,7 @@ class TDTestCase:
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}") tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor()) tdSql.init(conn.cursor())
self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 100 stt_trigger 1; ;use deldata;
create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int);
create table deldata.ct1 using deldata.stb1 tags ( 1 ); create table deldata.ct1 using deldata.stb1 tags ( 1 );
insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );

2
tests/system-test/0-others/udfTest.py
View File

@ -61,7 +61,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db ") tdSql.execute("drop database if exists db ")
tdSql.execute("create database if not exists db duration 300") tdSql.execute("create database if not exists db duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/0-others/udf_cfg1.py
View File

@ -63,7 +63,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db ") tdSql.execute("drop database if exists db ")
tdSql.execute("create database if not exists db duration 300") tdSql.execute("create database if not exists db duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/0-others/udf_cfg2.py
View File

@ -63,7 +63,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db ") tdSql.execute("drop database if exists db ")
tdSql.execute("create database if not exists db duration 300") tdSql.execute("create database if not exists db duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/0-others/udf_cluster.py
View File

@ -64,7 +64,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db") tdSql.execute("drop database if exists db")
tdSql.execute("create database if not exists db replica 1 duration 300") tdSql.execute("create database if not exists db replica 1 duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/0-others/udf_create.py
View File

@ -73,7 +73,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db ") tdSql.execute("drop database if exists db ")
tdSql.execute("create database if not exists db duration 300") tdSql.execute("create database if not exists db duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/0-others/udf_restart_taosd.py
View File

@ -60,7 +60,7 @@ class TDTestCase:
def prepare_data(self): def prepare_data(self):
tdSql.execute("drop database if exists db ") tdSql.execute("drop database if exists db ")
tdSql.execute("create database if not exists db duration 300") tdSql.execute("create database if not exists db duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/2-query/abs.py
View File

@ -127,7 +127,7 @@ class TDTestCase:
def prepare_tag_datas(self, dbname="testdb"): def prepare_tag_datas(self, dbname="testdb"):
# prepare datas # prepare datas
tdSql.execute( tdSql.execute(
f"create database if not exists {dbname} keep 3650 duration 1000 replica {self.replicaVar} ") f"create database if not exists {dbname} keep 3650 duration 100 replica {self.replicaVar} ")
tdSql.execute(" use testdb ") tdSql.execute(" use testdb ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/and_or_for_byte.py
View File

@ -128,7 +128,7 @@ class TDTestCase:
def prepare_tag_datas(self, dbname="testdb"): def prepare_tag_datas(self, dbname="testdb"):
# prepare datas # prepare datas
tdSql.execute( tdSql.execute(
f"create database if not exists {dbname} keep 3650 duration 1000") f"create database if not exists {dbname} keep 3650 duration 100")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/countAlwaysReturnValue.py
View File

@ -18,7 +18,7 @@ class TDTestCase:
def prepare_data(self, dbname="db"): def prepare_data(self, dbname="db"):
tdSql.execute( tdSql.execute(
f"create database if not exists {dbname} keep 3650 duration 1000") f"create database if not exists {dbname} keep 3650 duration 100")
tdSql.execute(f"use {dbname} ") tdSql.execute(f"use {dbname} ")
tdSql.execute( tdSql.execute(
f"create table {dbname}.tb (ts timestamp, c0 int)" f"create table {dbname}.tb (ts timestamp, c0 int)"

8
tests/system-test/2-query/db.py
View File

@ -57,11 +57,17 @@ class TDTestCase:
tdSql.checkData(0, 2, 0) tdSql.checkData(0, 2, 0)
tdSql.query("show dnode 1 variables like '%debugFlag'") tdSql.query("show dnode 1 variables like '%debugFlag'")
tdSql.checkRows(24) tdSql.checkRows(25)
tdSql.query("show dnode 1 variables like '____debugFlag'") tdSql.query("show dnode 1 variables like '____debugFlag'")
tdSql.checkRows(2) tdSql.checkRows(2)
tdSql.query("show dnode 1 variables like 's3MigrateEna%'")
tdSql.checkRows(1)
tdSql.checkData(0, 0, 1)
tdSql.checkData(0, 1, 's3MigrateEnabled')
tdSql.checkData(0, 2, 0)
def threadTest(self, threadID): def threadTest(self, threadID):
print(f"Thread {threadID} starting...") print(f"Thread {threadID} starting...")
tdsqln = tdCom.newTdSql() tdsqln = tdCom.newTdSql()

2
tests/system-test/2-query/distribute_agg_apercentile.py
View File

@ -18,7 +18,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 s3_keeplocal 3000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 s3_keeplocal 3000 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_avg.py
View File

@ -35,7 +35,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_count.py
View File

@ -36,7 +36,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_max.py
View File

@ -38,7 +38,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_min.py
View File

@ -37,7 +37,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname} ") tdSql.execute(f" use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_spread.py
View File

@ -37,7 +37,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname}") tdSql.execute(f" use {dbname}")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_stddev.py
View File

@ -46,7 +46,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname}") tdSql.execute(f" use {dbname}")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/distribute_agg_sum.py
View File

@ -35,7 +35,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f" use {dbname}") tdSql.execute(f" use {dbname}")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

4
tests/system-test/2-query/function_null.py
View File

@ -23,7 +23,7 @@ class TDTestCase:
def prepare_tag_datas(self, dbname="testdb"): def prepare_tag_datas(self, dbname="testdb"):
# prepare datas # prepare datas
tdSql.execute( tdSql.execute(
f"create database if not exists {dbname} keep 3650 duration 1000") f"create database if not exists {dbname} keep 3650 duration 100")
tdSql.execute(f"use {dbname} ") tdSql.execute(f"use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1
@ -249,4 +249,4 @@ class TDTestCase:
tdCases.addLinux(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase())

2
tests/system-test/2-query/irate.py
View File

@ -78,7 +78,7 @@ class TDTestCase:
def prepare_tag_datas(self, dbname="testdb"): def prepare_tag_datas(self, dbname="testdb"):
# prepare datas # prepare datas
tdSql.execute( tdSql.execute(
f"create database if not exists {dbname} keep 3650 duration 1000") f"create database if not exists {dbname} keep 3650 duration 100")
tdSql.execute(f"use {dbname} ") tdSql.execute(f"use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/join.py
View File

@ -370,7 +370,7 @@ class TDTestCase:
tdLog.printNoPrefix("==========step4:cross db check") tdLog.printNoPrefix("==========step4:cross db check")
dbname1 = "db1" dbname1 = "db1"
tdSql.execute(f"create database {dbname1} duration 432000m") tdSql.execute(f"create database {dbname1} duration 172800m")
tdSql.execute(f"use {dbname1}") tdSql.execute(f"use {dbname1}")
self.__create_tb(dbname=dbname1) self.__create_tb(dbname=dbname1)
self.__insert_data(dbname=dbname1) self.__insert_data(dbname=dbname1)

6
tests/system-test/2-query/last_row.py
View File

@ -61,7 +61,7 @@ class TDTestCase:
def prepare_datas(self ,cache_value, dbname="db"): def prepare_datas(self ,cache_value, dbname="db"):
tdSql.execute(f"drop database if exists {dbname} ") tdSql.execute(f"drop database if exists {dbname} ")
create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel {cache_value}" create_db_sql = f"create database if not exists {dbname} keep 3650 duration 100 cachemodel {cache_value}"
tdSql.execute(create_db_sql) tdSql.execute(create_db_sql)
tdSql.execute(f"use {dbname}") tdSql.execute(f"use {dbname}")
@ -129,7 +129,7 @@ class TDTestCase:
tdSql.execute(f"drop database if exists {dbname} ") tdSql.execute(f"drop database if exists {dbname} ")
# prepare datas # prepare datas
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel {cache_value}") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 cachemodel {cache_value}")
tdSql.execute(f"use {dbname} ") tdSql.execute(f"use {dbname} ")
@ -871,7 +871,7 @@ class TDTestCase:
def initLastRowDelayTest(self, dbname="db"): def initLastRowDelayTest(self, dbname="db"):
tdSql.execute(f"drop database if exists {dbname} ") tdSql.execute(f"drop database if exists {dbname} ")
create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel 'NONE' REPLICA 1" create_db_sql = f"create database if not exists {dbname} keep 3650 duration 100 cachemodel 'NONE' REPLICA 1"
tdSql.execute(create_db_sql) tdSql.execute(create_db_sql)
time.sleep(3) time.sleep(3)

2
tests/system-test/2-query/max.py
View File

@ -117,7 +117,7 @@ class TDTestCase:
def support_distributed_aggregate(self, dbname="testdb"): def support_distributed_aggregate(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute(f"use {dbname} ") tdSql.execute(f"use {dbname} ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1

2
tests/system-test/2-query/sample.py
View File

@ -611,7 +611,7 @@ class TDTestCase:
def basic_sample_query(self, dbname="db"): def basic_sample_query(self, dbname="db"):
tdSql.execute(f" drop database if exists {dbname} ") tdSql.execute(f" drop database if exists {dbname} ")
tdSql.execute(f" create database if not exists {dbname} duration 300d ") tdSql.execute(f" create database if not exists {dbname} duration 120d ")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)

29
tests/system-test/2-query/tsma.py
View File

@ -693,7 +693,7 @@ class TDTestCase:
"======== prepare test env include database, stable, ctables, and insert data: ") "======== prepare test env include database, stable, ctables, and insert data: ")
paraDict = {'dbName': db, paraDict = {'dbName': db,
'dropFlag': 1, 'dropFlag': 1,
'vgroups': 2, 'vgroups': 4,
'stbName': 'meters', 'stbName': 'meters',
'colPrefix': 'c', 'colPrefix': 'c',
'tagPrefix': 't', 'tagPrefix': 't',
@ -1273,6 +1273,21 @@ class TDTestCase:
else: else:
tdLog.debug(f'wait query succeed: {sql} to return {expected_row_num}, got: {tdSql.getRows()}') tdLog.debug(f'wait query succeed: {sql} to return {expected_row_num}, got: {tdSql.getRows()}')
def wait_query_err(self, sql: str, timeout_in_seconds: float, err):
timeout = timeout_in_seconds
while timeout > 0:
try:
tdSql.query(sql, queryTimes=1)
time.sleep(1)
timeout = timeout - 1
except:
tdSql.error(sql, err);
break
if timeout <= 0:
tdLog.exit(f'failed to wait query: {sql} to return error timeout: {timeout_in_seconds}s')
else:
tdLog.debug(f'wait query error succeed: {sql}')
def test_drop_tsma(self): def test_drop_tsma(self):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
tdLog.debug(f'-----{function_name}------') tdLog.debug(f'-----{function_name}------')
@ -1338,15 +1353,15 @@ class TDTestCase:
self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m') self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m')
tdSql.execute('alter table test.t0 ttl 2', queryTimes=1) tdSql.execute('alter table test.t0 ttl 2', queryTimes=1)
tdSql.execute('flush database test') tdSql.execute('flush database test')
self.wait_query('show test.tables like "%t0"', 0, wait_query_seconds) res_tb = TSMAQCBuilder().md5('1.test.tsma1_t0')
self.wait_query_err(f'desc test.`{res_tb}`', wait_query_seconds, -2147473917)
# test drop multi tables # test drop multi tables
tdSql.execute('drop table test.t3, test.t4') tdSql.execute('drop table test.t3, test.t4')
self.wait_query('show test.tables like "%t3"', 0, wait_query_seconds) res_tb = TSMAQCBuilder().md5('1.test.tsma1_t3')
self.wait_query('show test.tables like "%t4"', 0, wait_query_seconds) self.wait_query_err(f'desc test.`{res_tb}`', wait_query_seconds, -2147473917)
res_tb = TSMAQCBuilder().md5('1.test.tsma1_t4')
tdSql.query('show test.tables like "%tsma%"') self.wait_query_err(f'desc test.`{res_tb}`', wait_query_seconds, -2147473917)
tdSql.checkRows(0)
# test drop stream # test drop stream
tdSql.error('drop stream tsma1', -2147471088) ## TSMA must be dropped first tdSql.error('drop stream tsma1', -2147471088) ## TSMA must be dropped first

2
tests/system-test/2-query/twa.py
View File

@ -22,7 +22,7 @@ class TDTestCase:
def prepare_datas_of_distribute(self, dbname="testdb"): def prepare_datas_of_distribute(self, dbname="testdb"):
# prepate datas for 20 tables distributed at different vgroups # prepate datas for 20 tables distributed at different vgroups
tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 100 vgroups 5")
tdSql.execute( tdSql.execute(
f'''create table {dbname}.stb1 f'''create table {dbname}.stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp,c11 int UNSIGNED, c12 bigint UNSIGNED, c13 smallint UNSIGNED, c14 tinyint UNSIGNED) (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp,c11 int UNSIGNED, c12 bigint UNSIGNED, c13 smallint UNSIGNED, c14 tinyint UNSIGNED)

2
tests/system-test/6-cluster/5dnode1mnode.py
View File

@ -110,7 +110,7 @@ class TDTestCase:
tdSql.error("drop mnode on dnode 1;") tdSql.error("drop mnode on dnode 1;")
tdSql.execute("drop database if exists db") tdSql.execute("drop database if exists db")
tdSql.execute("create database if not exists db replica 1 duration 300") tdSql.execute("create database if not exists db replica 1 duration 100")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

2
tests/system-test/6-cluster/5dnode2mnode.py
View File

@ -84,7 +84,7 @@ class TDTestCase:
# fisrt add data : db\stable\childtable\general table # fisrt add data : db\stable\childtable\general table
tdSql.execute("drop database if exists db2") tdSql.execute("drop database if exists db2")
tdSql.execute("create database if not exists db2 replica 1 duration 300") tdSql.execute("create database if not exists db2 replica 1 duration 100")
tdSql.execute("use db2") tdSql.execute("use db2")
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

4
tests/system-test/6-cluster/5dnode3mnodeAdd1Ddnoe.py
View File

@ -73,8 +73,8 @@ class TDTestCase:
for couti in range(countstart,countstop): for couti in range(countstart,countstop):
tdLog.debug("drop database if exists db%d" %couti) tdLog.debug("drop database if exists db%d" %couti)
tdSql.execute("drop database if exists db%d" %couti, 20) tdSql.execute("drop database if exists db%d" %couti, 20)
print("create database if not exists db%d replica 1 duration 300" %couti) print("create database if not exists db%d replica 1 duration 100" %couti)
tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) tdSql.execute("create database if not exists db%d replica 1 duration 100" %couti)
tdSql.execute("use db%d" %couti) tdSql.execute("use db%d" %couti)
tdSql.execute( tdSql.execute(
'''create table stb1 '''create table stb1

Some files were not shown because too many files have changed in this diff Show More