Merge branch 'main' into doc/contrib
This commit is contained in:
Shengliang Guan
commit
25816f7665
|
|
@ -0,0 +1,66 @@
|
|||
name: TDengine Build
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- 'main'
|
||||
- '3.0'
|
||||
- '3.1'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Run unit tests
|
||||
|
||||
steps:
|
||||
- name: Checkout the repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 1.18
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
sudo apt update -y
|
||||
sudo apt install -y build-essential cmake \
|
||||
libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev \
|
||||
zlib1g pkg-config libssl-dev gawk
|
||||
|
||||
|
||||
- name: Build and install TDengine
|
||||
run: |
|
||||
mkdir debug && cd debug
|
||||
cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false \
|
||||
-DBUILD_TOOLS=true -DBUILD_TEST=off \
|
||||
-DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false
|
||||
make -j 4
|
||||
sudo make install
|
||||
which taosd
|
||||
which taosadapter
|
||||
which taoskeeper
|
||||
|
||||
- name: Start taosd
|
||||
run: |
|
||||
cp /etc/taos/taos.cfg ./
|
||||
sudo echo "supportVnodes 256" >> taos.cfg
|
||||
nohup sudo taosd -c taos.cfg &
|
||||
|
||||
- name: Start taosadapter
|
||||
run: nohup sudo taosadapter &
|
||||
|
||||
- name: Run tests with taosBenchmark
|
||||
run: |
|
||||
taosBenchmark -t 10 -n 10 -y
|
||||
taos -s "select count(*) from test.meters"
|
||||
|
||||
- name: Clean up
|
||||
if: always()
|
||||
run: |
|
||||
if pgrep taosd; then sudo pkill taosd; fi
|
||||
if pgrep taosadapter; then sudo pkill taosadapter; fi
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
</p>
|
||||
<p>
|
||||
|
||||
[](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml)
|
||||
[](https://coveralls.io/github/taosdata/TDengine?branch=3.0)
|
||||
[](https://bestpractices.coreinfrastructure.org/projects/4201)
|
||||
<br />
|
||||
|
|
|
|||
|
|
@ -190,7 +190,8 @@ The effective value of charset is UTF-8.
|
|||
|Parameter Name |Supported Version |Dynamic Modification|Description|
|
||||
|-----------------------|-------------------------|--------------------|------------|
|
||||
|supportVnodes | |Supported, effective immediately |Maximum number of vnodes supported by a dnode, range 0-4096, default value is twice the number of CPU cores + 5|
|
||||
|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 0-1024, default value 4|
|
||||
|numOfCommitThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-1024, default value 4|
|
||||
|numOfCompactThreads | |Supported, effective after restart|Maximum number of commit threads, range 1-16, default value 2|
|
||||
|numOfMnodeReadThreads | |Supported, effective after restart|Number of Read threads for mnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)|
|
||||
|numOfVnodeQueryThreads | |Supported, effective after restart|Number of Query threads for vnode, range 0-1024, default value is twice the number of CPU cores (not exceeding 16)|
|
||||
|numOfVnodeFetchThreads | |Supported, effective after restart|Number of Fetch threads for vnode, range 0-1024, default value is one quarter of the CPU cores (not exceeding 4)|
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ When using time windows, note:
|
|||
- The window width of the aggregation period is specified by the keyword INTERVAL, with the shortest interval being 10 milliseconds (10a); it also supports an offset (the offset must be less than the interval), which is the offset of the time window division compared to "UTC moment 0". The SLIDING statement is used to specify the forward increment of the aggregation period, i.e., the duration of each window slide forward.
|
||||
- When using the INTERVAL statement, unless in very special cases, it is required to configure the timezone parameter in the taos.cfg configuration files of both the client and server to the same value to avoid frequent cross-time zone conversions by time processing functions, which can cause severe performance impacts.
|
||||
- The returned results have a strictly monotonically increasing time-series.
|
||||
- When using AUTO as the window offset, if the WHERE time condition is complex, such as multiple AND/OR/IN combinations, AUTO may not take effect. In such cases, you can manually specify the window offset to resolve the issue.
|
||||
- When using AUTO as the window offset, if the window width unit is d (day), n (month), w (week), y (year), such as: INTERVAL(1d, AUTO), INTERVAL(3w, AUTO), the TSMA optimization cannot take effect. If TSMA is manually created on the target table, the statement will report an error and exit; in this case, you can explicitly specify the Hint SKIP_TSMA or not use AUTO as the window offset.
|
||||
|
||||
### State Window
|
||||
|
|
|
|||
|
|
@ -166,6 +166,12 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
|
|||
|
||||
|
||||
|
||||
### 8. 创建完成
|
||||
### 8. 异常处理策略
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
<Contributing />
|
||||
|
||||
### 9. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 MQTT 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -196,12 +196,16 @@ json 数据支持 JSONObject 或者 JSONArray，使用 json 解析器可以解
|
|||
|
||||
### 8. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
|
||||
<AdvancedOptions/>
|
||||
|
||||
|
||||
### 9. 异常处理策略
|
||||
|
||||
### 9. 创建完成
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
<Contributing />
|
||||
|
||||
### 10. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 Kafka 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -75,9 +75,9 @@ InfluxDB 是一种流行的开源时间序列数据库，它针对处理大量
|
|||
|
||||
### 6. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
|
||||
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
<AdvancedOptions/>
|
||||
|
||||
### 7. 创建完成
|
||||
|
||||
|
|
@ -58,9 +58,9 @@ OpenTSDB 是一个架构在 HBase 系统之上的实时监控信息收集和展
|
|||
|
||||
### 5. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
|
||||
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
<AdvancedOptions/>
|
||||
|
||||
### 6. 创建完成
|
||||
|
||||
|
|
@ -107,13 +107,25 @@ sidebar_label: "CSV"
|
|||
|
||||
|
||||
|
||||
### 5. 创建完成
|
||||
### 5. 配置高级选项
|
||||
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
<AdvancedOptions/>
|
||||
|
||||
### 6. 异常处理策略
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
<Contributing />
|
||||
|
||||
### 7. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 CSV 到 TDengine 的数据同步任务，回到数据写入任务列表页面，可查看任务执行情况，也可以进行任务的“启动/停止”操作与“查看/编辑/删除/复制”操作。
|
||||
|
||||
|
||||
|
||||
### 6. 查看运行指标
|
||||
### 8. 查看运行指标
|
||||
|
||||
点击 **查看** 按钮，查看任务的运行指标，同时也可以查看任务中所有文件的处理情况。
|
||||
|
||||
|
|
@ -134,6 +134,12 @@ split 提取器，seperator 填写分割符 `,`, number 填写 2。
|
|||
|
||||
|
||||
|
||||
### 7. 创建完成
|
||||
### 7. 异常处理策略
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
<Contributing />
|
||||
|
||||
### 8. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建任务。提交任务后，回到**数据写入**页面可以查看任务状态。
|
||||
|
|
@ -98,14 +98,16 @@ MySQL 是最流行的关系型数据库之一。很多系统都曾经或正在
|
|||
|
||||
### 8. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
<AdvancedOptions/>
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
### 9. 异常处理策略
|
||||
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
### 9. 创建完成
|
||||
<Contributing />
|
||||
|
||||
### 10. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 MySQL 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -99,14 +99,16 @@ TDengine 可以高效地从 PostgreSQL 读取数据并将其写入 TDengine，
|
|||
|
||||
### 8. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
<AdvancedOptions/>
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
### 9. 异常处理策略
|
||||
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
### 9. 创建完成
|
||||
<Contributing />
|
||||
|
||||
### 10. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 PostgreSQL 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -91,14 +91,16 @@ TDengine 可以高效地从 Oracle 读取数据并将其写入 TDengine，以实
|
|||
|
||||
### 7. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
<AdvancedOptions/>
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
### 8. 异常处理策略
|
||||
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
### 8. 创建完成
|
||||
<Contributing />
|
||||
|
||||
### 9. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 Oracle 到 TDengine 的数据同步任务，回到**数据源列表****页面可查看任务执行情况。
|
||||
|
|
@ -105,14 +105,16 @@ Microsoft SQL Server 是最流行的关系型数据库之一。很多系统都
|
|||
|
||||
### 8. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
<AdvancedOptions/>
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
### 9. 异常处理策略
|
||||
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
### 9. 创建完成
|
||||
<Contributing />
|
||||
|
||||
### 10. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 Microsoft SQL Server 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -122,14 +122,16 @@ MongoDB 是一个介于关系型数据库与非关系型数据库之间的产品
|
|||
|
||||
### 8. 配置高级选项
|
||||
|
||||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
import AdvancedOptions from './_02-advanced_options.mdx'
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
<AdvancedOptions/>
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
### 9. 异常处理策略
|
||||
|
||||
|
||||
import Contributing from './_03-exception-handling-strategy.mdx'
|
||||
|
||||
### 9. 创建完成
|
||||
<Contributing />
|
||||
|
||||
### 10. 创建完成
|
||||
|
||||
点击 **提交** 按钮，完成创建 MongoDB 到 TDengine 的数据同步任务，回到**数据源列表**页面可查看任务执行情况。
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
**高级选项** 区域是默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
|
||||
**最大读取并发数** 数据源连接数或读取线程数限制，当默认参数不满足需要或需要调整资源使用量时修改此参数。
|
||||
|
||||
**批次大小** 单次发送的最大消息数或行数。默认是 10000。
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
异常处理策略区域是对数据异常时的处理策略进行配置，默认折叠的，点击右侧 `>` 可以展开，如下图所示：
|
||||
|
||||
|
||||
|
||||
各异常项说明及相应可选处理策略如下：
|
||||
|
||||
> 通用处理策略说明：
|
||||
> 归档：将异常数据写入归档文件（默认路径为 `${data_dir}/tasks/_id/.datetime`），不写入目标库
|
||||
> 丢弃：将异常数据忽略，不写入目标库
|
||||
> 报错：任务报错
|
||||
|
||||
- **主键时间戳溢出** 检查数据中第一列时间戳是否在正确的时间范围内（now - keep1, now + 100y），可选处理策略：归档、丢弃、报错
|
||||
- **主键时间戳空** 检查数据中第一列时间戳是否为空，可选处理策略：归档、丢弃、报错、使用当前时间
|
||||
> 使用当前时间：使用当前时间填充到空的时间戳字段中
|
||||
- **表名长度溢出** 检查子表表名的长度是否超出限制（最大 192 字符），可选处理策略：归档、丢弃、报错、截断、截断且归档
|
||||
> 截断：截取原始表名的前 192 个字符作为新的表名
|
||||
> 截断且归档：截取原始表名的前 192 个字符作为新的表名，并且将此行记录写入归档文件
|
||||
- **表名非法字符** 检查子表表名中是否包含特殊字符（符号 `.` 等），可选处理策略：归档、丢弃、报错、非法字符替换为指定字符串
|
||||
> 非法字符替换为指定字符串：将原始表名中的特殊字符替换为后方输入框中的指定字符串，例如 `a.b` 替换为 `a_b`
|
||||
- **表名模板变量空值** 检查子表表名模板中的变量是否为空，可选处理策略：丢弃、留空、变量替换为指定字符串
|
||||
> 留空：变量位置不做任何特殊处理，例如 `a_{x}` 转换为 `a_`
|
||||
> 变量替换为指定字符串：变量位置使用后方输入框中的指定字符串，例如 `a_{x}` 转换为 `a_b`
|
||||
- **列名长度溢出** 检查列名的长度是否超出限制（最大 64 字符），可选处理策略：归档、丢弃、报错
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.7 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 98 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.2 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 31 KiB |
|
|
@ -145,3 +145,47 @@ toasX 的配置文件(默认 /etc/taos/taosx.toml) 中与 monitor 相关的配
|
|||
#### 限制
|
||||
|
||||
只有在以 server 模式运行 taosX 时，与监控相关的配置才生效。
|
||||
|
||||
## explorer 集成监控面板
|
||||
|
||||
explorer 支持集成已有的 grafana dashboard。
|
||||
|
||||
### 配置 grafana
|
||||
|
||||
编辑 grafana.ini, 修改以下配置项。配置 root_url, 可能对现有的 grafana 使用习惯有所影响，为了集成到 explorer 是需要如此配置的, 方便通过 explorer 做服务代理。
|
||||
|
||||
``` toml
|
||||
[server]
|
||||
# If you use reverse proxy and sub path specify full url (with sub path)
|
||||
root_url = http://ip:3000/grafana
|
||||
# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
|
||||
serve_from_sub_path = true
|
||||
|
||||
[security]
|
||||
# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
|
||||
allow_embedding = true
|
||||
```
|
||||
|
||||
### 配置 Explorer
|
||||
|
||||
修改 explorer.toml, 其中 dashboard 配置的 url 中的 ip, 应该配置为可以通过 explorer 服务器能够访问到的 grafana 服务的内网地址。
|
||||
|
||||
``` toml
|
||||
[grafana]
|
||||
# The token of the Grafana server, which is used to access the Grafana server.
|
||||
token = ""
|
||||
|
||||
# The URL of the Grafana dashboard, which is used to display the monitoring data of the TDengine cluster.
|
||||
# You can configure multiple Grafana dashboards.
|
||||
[grafana.dashboards]
|
||||
TDengine3 = "http://ip:3000/d/000000001/tdengine3?theme=light&kiosk=tv"
|
||||
taosX = "http://ip:3000/d/000000002/taosx?theme=light&kiosk=tv"
|
||||
```
|
||||
|
||||
如下图(grafana V-8.5.27)，获取 api key, 请注意添加只读权限的 apikey, 否则有安全风险。
|
||||
|
||||
|
||||
|
||||
如下图(grafana V-8.5.27)，获取 dashboard url, 获取的 url 请额外加上参数：theme=light&kiosk=tv.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -35,119 +35,67 @@ taosdump -i /file/path -h localhost -P 6030
|
|||
|
||||
# 2. 基于 TDengine Enterprise 进行数据备份恢复
|
||||
|
||||
## 2.1. 概念
|
||||
## 2.1. 概述
|
||||
|
||||
基于 TDengine 的数据订阅功能，TDengine Enterprise 实现了数据的增量备份和恢复。用户可以通过 taosExplorer 对 TDengine
|
||||
集群进行备份和恢复。
|
||||
|
||||
TDengine Enterprise 的备份和恢复功能包括以下几个概念：
|
||||
|
||||
1. 备份对象：用户可以对一个数据库，或者一个超级表进行备份。
|
||||
2. 备份计划：用户可以为某个备份对象创建一个备份计划。备份计划从指定的时间点开始，周期性的执行一次备份任务，并生成一组备份文件。
|
||||
3. 备份点：每执行一次备份任务，生成一组备份文件，它们对应一个时间点，称为**备份点**。第一个备份点称为**初始备份点**。
|
||||
4. 备份文件：多个备份点，组成备份计划的备份文件。
|
||||
5. 恢复任务：用户可以选择备份计划的某个备份点，创建一个恢复任务。恢复任务会从初始备份点开始，逐个应用备份点，恢复到指定的备份点。
|
||||
1. 增量数据备份：基于 TDengine 的数据订阅功能，将**备份对象**的所有数据变更（包括：新增、修改、删除、元数据变更等）记录下来，生成备份文件。
|
||||
2. 数据恢复：使用增量数据备份生成的备份文件，将**备份对象**恢复到指定的时间点。
|
||||
3. 备份对象：用户备份的对象，可以是一个**数据库**，也可以是一个**超级表**。
|
||||
4. 备份计划：用户为备份对象创建一个周期性执行的备份任务。备份计划从指定的时间点开始，以**备份周期**为间隔，周期性地执行备份任务。备份任务每次生成一个**备份点**。
|
||||
5. 备份点：每次执行备份任务，生成一组备份文件，它们对应一个时间点，称为**备份点**。第一个备份点称为**初始备份点**。
|
||||
6. 恢复任务：用户选择备份计划的某个备份点，创建一个恢复任务。恢复任务从**初始备份点**开始，逐个回放**备份文件**中的数据变更，直到指定的备份点结束。
|
||||
|
||||
|
||||
|
||||
以上面的图为例：
|
||||
以上图为例：
|
||||
|
||||
1. 用户创建了一个备份计划，从 2024-08-27 00:00:00 开始，每隔 1 天执行一次备份任务。
|
||||
2. 在 2024-08-27 00:00:00 执行了第一次备份任务，生成了一个备份点。
|
||||
3. 之后，每隔 1 天执行一次备份任务，生成了多个备份点，组成了备份文件。
|
||||
4. 用户可以选择某个备份点，创建一个恢复任务，恢复到指定的备份点。
|
||||
1. 用户创建了一个**备份计划**，从 2024-08-27 00:00:00 开始，每隔 1 天执行一次**备份任务**。
|
||||
2. 在 2024-08-27 00:00:00 执行了第一次备份任务，生成了一个**初始备份点**。
|
||||
3. 之后，每隔 1 天执行一次备份任务，生成了多个**备份点**。
|
||||
4. 用户可以选择某个**备份点**，创建一个**恢复任务**。
|
||||
5. 恢复任务会从初始备份点开始，逐个应用备份点，恢复到指定的备份点。
|
||||
|
||||
## 2.2. 备份计划
|
||||
## 2.2. 数据备份
|
||||
|
||||
### 2.1.1. 创建
|
||||
|
||||
1. 通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。
|
||||
2. 在 taosExplorer 服务页面中，进入“系统管理 - 备份”页面，点击“创建备份计划”按钮。
|
||||
|
||||
|
||||
|
||||
3. 在弹出的“创建备份计划”表单中，填写备份计划的相关信息。
|
||||
|
||||
|
||||
通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 在
|
||||
taosExplorer 服务页面中，进入“系统管理 - 备份”页面，在“备份计划”标签页下，点击“创建备份计划”，填写备份计划的相关信息。
|
||||
|
||||
需要填写的信息包括：
|
||||
|
||||
* 数据库：需要备份的数据库名称。一个备份计划只能备份一个数据库/超级表。
|
||||
* 超级表：需要备份的超级表名称。如果不填写，则备份整个数据库。
|
||||
* 下次执行时间：首次执行备份任务的日期时间。
|
||||
* 备份周期：备份点之间的时间间隔。注意：备份周期必须大于数据库的 WAL_RETENTION_PERIOD 参数值。
|
||||
* 错误重试次数：对于可通过重试解决的错误，系统会按照此次数进行重试。
|
||||
* 错误重试间隔：每次重试之间的时间间隔。
|
||||
* 目录：存储备份文件的目录。
|
||||
* 备份文件大小：备份文件的大小限制。当备份文件大小达到此限制时，会自动创建新的备份文件。
|
||||
* 文件压缩等级：备份文件的压缩等级。支持：最快速度、最佳压缩比、兼具速度和压缩比。
|
||||
1. 数据库：需要备份的数据库名称。一个备份计划只能备份一个数据库/超级表。
|
||||
2. 超级表：需要备份的超级表名称。如果不填写，则备份整个数据库。
|
||||
3. 下次执行时间：首次执行备份任务的日期时间。
|
||||
4. 备份周期：备份点之间的时间间隔。注意：备份周期必须大于数据库的 WAL_RETENTION_PERIOD 参数值。
|
||||
5. 错误重试次数：对于可通过重试解决的错误，系统会按照此次数进行重试。
|
||||
6. 错误重试间隔：每次重试之间的时间间隔。
|
||||
7. 目录：存储备份文件的目录。
|
||||
8. 备份文件大小：备份文件的大小限制。当备份文件大小达到此限制时，会自动创建新的备份文件。
|
||||
9. 文件压缩等级：备份文件的压缩等级。支持：最快速度、最佳压缩比、兼具速度和压缩比。
|
||||
|
||||
创建成功后，备份计划会开始按照配置的参数运行。
|
||||
创建成功后，备份计划会开始按照配置的参数运行。在“备份计划”下的列表中，可以查看已创建的备份计划。
|
||||
|
||||
### 2.1.2. 查看
|
||||
备份计划支持以下操作：
|
||||
|
||||
在“备份计划”下的列表中，可以查看已创建的备份计划。
|
||||
1. 查看：显示备份计划的详细信息。
|
||||
2. 修改：修改备份计划的配置。修改备份计划的配置后，当前运行的备份任务会先停止，然后按照新的配置重新运行。
|
||||
3. 复制：以选中的备份计划为模版，创建新的备份计划。除了数据库和超级表需要用户选择以外，其他配置项和被复制的计划相同。
|
||||
4. 删除：删除备份计划。删除备份计划时，可以选择是否删除关联的备份文件。
|
||||
5. 指标：查看备份计划的统计指标。
|
||||
6. 查看备份点：查看和备份计划关联的所有备份点。
|
||||
|
||||
|
||||
## 2.3. 备份文件
|
||||
|
||||
点击“操作”中的“查看”按钮，可以查看备份计划的详细信息。
|
||||
在“备份文件”列表中，可以查看备份文件的详细信息。
|
||||
|
||||
|
||||
## 2.4. 数据恢复
|
||||
|
||||
### 2.1.3. 修改
|
||||
在“备份文件”列表中，选择一个备份点，可以创建一个恢复任务，数据库恢复到指定的时间。
|
||||
|
||||
点击“操作”中的“修改”按钮，可以修改备份计划的配置。
|
||||
|
||||
|
||||
|
||||
修改备份计划的配置后，当前运行的备份任务会先停止，然后按照新的配置重新运行。
|
||||
|
||||
### 2.1.4. 复制
|
||||
|
||||
点击“操作”中的“复制”按钮，可以复制备份计划。
|
||||
|
||||
|
||||
|
||||
除了数据库和超级表被置为空外，其他配置项和被复制的计划相同。用户点击“确认”后，创建一个新的备份计划。
|
||||
|
||||
### 2.1.5. 删除
|
||||
|
||||
在操作中点击关闭按钮，可以停止当前备份计划。点击“操作”中的“删除”按钮，可以删除备份计划。
|
||||
|
||||
|
||||
|
||||
删除备份计划时，可以选择，是否删除关联的备份文件。
|
||||
|
||||
## 2.2. 备份文件
|
||||
|
||||
### 2.2.1. 查看
|
||||
|
||||
在备份计划列表中，选择要一个备份计划。在“备份文件”列中，点击“查看”按钮。可以查看和备份计划的所有备份点。
|
||||
|
||||
|
||||
|
||||
在备份文件列表中，可以查看备份文件的详细信息。
|
||||
|
||||
|
||||
|
||||
## 2.3. 恢复任务
|
||||
|
||||
### 2.3.1. 创建
|
||||
|
||||
在备份文件列表中，点击“操作”中的“恢复”按钮，可以创建一个恢复任务。
|
||||
|
||||
|
||||
|
||||
在弹出的对话框中，选择使用哪个备份点开始恢复，默认为最早的备份点。点击“确定”后，创建恢复任务，并跳转至“恢复任务”列表。
|
||||
|
||||
### 2.3.2. 查看
|
||||
|
||||
在“恢复任务”列表中，可以查看已创建的恢复任务。
|
||||
|
||||
|
||||
|
||||
恢复任务可以终止。点击“操作”中的开关，可以终止当前恢复任务。
|
||||
在“恢复任务”列表中，可以查看已创建的恢复任务。恢复任务可以终止。
|
||||
|
||||
# 3. 常见错误排查
|
||||
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 32 KiB After Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 94 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 55 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 119 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 50 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 109 KiB |
|
|
@ -374,7 +374,8 @@ charset 的有效值是 UTF-8。
|
|||
|参数名称|支持版本|动态修改|参数含义|
|
||||
|--------------------------|----------|-------------------------|-|
|
||||
|supportVnodes | |支持动态修改 立即生效 |dnode 支持的最大 vnode 数目，取值范围 0-4096，默认值 CPU 核数的 2 倍 + 5|
|
||||
|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 0-1024，默认值为 4|
|
||||
|numOfCommitThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-1024，默认值为 4|
|
||||
|numOfCompactThreads | |支持动态修改 重启生效 |落盘线程的最大数量，取值范围 1-16，默认值为 2|
|
||||
|numOfMnodeReadThreads | |支持动态修改 重启生效 |mnode 的 Read 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|
||||
|numOfVnodeQueryThreads | |支持动态修改 重启生效 |vnode 的 Query 线程数目，取值范围 0-1024，默认值为 CPU 核数的两倍（不超过 16）|
|
||||
|numOfVnodeFetchThreads | |支持动态修改 重启生效 |vnode 的 Fetch 线程数目，取值范围 0-1024，默认值为 CPU 核数的四分之一（不超过 4）|
|
||||
|
|
@ -439,7 +440,7 @@ charset 的有效值是 UTF-8。
|
|||
|minimalLogDirGB | ｜不支持动态修改 |日志文件夹所在磁盘可用空间大小小于该值时，停止写日志，单位 GB，默认值 1|
|
||||
|numOfLogLines | ｜支持动态修改 立即生效 |单个日志文件允许的最大行数，默认值 10,000,000|
|
||||
|asyncLog | |支持动态修改 立即生效 |日志写入模式，0：同步，1：异步，默认值 1|
|
||||
|logKeepDays | ｜支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，默认值 0，意味着无限保存，日志文件不会被重命名，也不会有新的日志文件滚动产生，但日志文件的内容有可能会不断滚动，取决于日志文件大小的设置；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件|
|
||||
|logKeepDays | ｜支持动态修改 立即生效 |日志文件的最长保存时间，单位：天，默认值 0，小于等于0意味着只有两个日志文件相互切换保存日志，超过两个文件保存数量的日志会被删除；当设置为大于 0 的值时，当日志文件大小达到设置的上限时会被重命名为 taosdlog.yyy，其中 yyy 为日志文件最后修改的时间戳，并滚动产生新的日志文件|
|
||||
|slowLogThreshold|3.3.3.0 后|支持动态修改 立即生效 |慢查询门限值，大于等于门限值认为是慢查询，单位秒，默认值 3 |
|
||||
|slowLogMaxLen |3.3.3.0 后|支持动态修改 立即生效 |慢查询日志最大长度，取值范围 1-16384，默认值 4096|
|
||||
|slowLogScope |3.3.3.0 后|支持动态修改 立即生效 |慢查询记录类型，取值范围 ALL/QUERY/INSERT/OTHERS/NONE，默认值 QUERY|
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ taosAdapter 是一个 TDengine 的配套工具，是 TDengine 集群和应用程
|
|||
|
||||
taosAdapter 提供以下功能：
|
||||
|
||||
- RESTful 接口
|
||||
- Websocket/RESTful 接口
|
||||
- 兼容 InfluxDB v1 写接口
|
||||
- 兼容 OpenTSDB JSON 和 telnet 格式写入
|
||||
- 无缝连接到 Telegraf
|
||||
|
|
|
|||
|
|
@ -138,6 +138,7 @@ SELECT COUNT(*) FROM meters WHERE _rowts - voltage > 1000000;
|
|||
- 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。
|
||||
- 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。
|
||||
- 返回的结果中时间序列严格单调递增。
|
||||
- 使用 AUTO 作为窗口偏移量时，如果 WHERE 时间条件比较复杂，比如多个 AND/OR/IN 互相组合，那么 AUTO 可能不生效，这种情况可以通过手动指定窗口偏移量进行解决。
|
||||
- 使用 AUTO 作为窗口偏移量时，如果窗口宽度的单位是 d (天), n (月), w (周), y (年)，比如: INTERVAL(1d, AUTO), INTERVAL(3w, AUTO)，此时 TSMA 优化无法生效。如果目标表上手动创建了TSMA，语句会报错退出；这种情况下，可以显式指定 Hint SKIP_TSMA 或者不使用 AUTO 作为窗口偏移量。
|
||||
|
||||
### 状态窗口
|
||||
|
|
|
|||
|
|
@ -182,7 +182,7 @@ void tColDataClear(SColData *pColData);
|
|||
void tColDataDeepClear(SColData *pColData);
|
||||
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
|
||||
int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward);
|
||||
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
|
||||
int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
|
||||
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal);
|
||||
int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg);
|
||||
void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key);
|
||||
|
|
|
|||
|
|
@ -112,7 +112,6 @@ extern int32_t tsNumOfSnodeWriteThreads;
|
|||
extern int64_t tsQueueMemoryAllowed;
|
||||
extern int32_t tsRetentionSpeedLimitMB;
|
||||
|
||||
extern const char *tsAlterCompactTaskKeywords;
|
||||
extern int32_t tsNumOfCompactThreads;
|
||||
extern int32_t tsNumOfRetentionThreads;
|
||||
|
||||
|
|
@ -291,8 +290,8 @@ extern int tsStreamAggCnt;
|
|||
extern bool tsFilterScalarMode;
|
||||
extern int32_t tsMaxStreamBackendCache;
|
||||
extern int32_t tsPQSortMemThreshold;
|
||||
extern int32_t tsResolveFQDNRetryTime;
|
||||
extern bool tsStreamCoverage;
|
||||
extern int8_t tsS3EpNum;
|
||||
|
||||
extern bool tsExperimental;
|
||||
// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
|
||||
|
|
|
|||
|
|
@ -389,7 +389,6 @@ typedef struct SStateStore {
|
|||
int32_t (*streamStateFillGetGroupKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
|
||||
int32_t (*streamStateGetKVByCur)(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
|
||||
|
||||
void (*streamStateSetFillInfo)(SStreamState* pState);
|
||||
void (*streamStateClearExpiredState)(SStreamState* pState);
|
||||
|
||||
int32_t (*streamStateSessionAddIfNotExist)(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal,
|
||||
|
|
@ -455,7 +454,6 @@ typedef struct SStateStore {
|
|||
int32_t (*streamStateBegin)(SStreamState* pState);
|
||||
void (*streamStateCommit)(SStreamState* pState);
|
||||
void (*streamStateDestroy)(SStreamState* pState, bool remove);
|
||||
int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
|
||||
void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
|
||||
void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
|
||||
} SStateStore;
|
||||
|
|
|
|||
|
|
@ -237,18 +237,20 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR
|
|||
(void)memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes);
|
||||
} else {
|
||||
int32_t dataLen = varDataTLen(pData);
|
||||
if (meta->type == TSDB_DATA_TYPE_JSON) {
|
||||
if (*pData == TSDB_DATA_TYPE_NULL) {
|
||||
dataLen = 0;
|
||||
} else if (*pData == TSDB_DATA_TYPE_NCHAR) {
|
||||
dataLen = varDataTLen(pData + sizeof(char));
|
||||
} else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
|
||||
dataLen = sizeof(int64_t);
|
||||
} else if (*pData == TSDB_DATA_TYPE_BOOL) {
|
||||
dataLen = sizeof(char);
|
||||
}
|
||||
dataLen += sizeof(char);
|
||||
}
|
||||
// This is a piece of code to help users implement udf. It is only called during testing.
|
||||
// Currently, the json type is not supported and will not be called.
|
||||
// if (meta->type == TSDB_DATA_TYPE_JSON) {
|
||||
// if (*pData == TSDB_DATA_TYPE_NULL) {
|
||||
// dataLen = 0;
|
||||
// } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
|
||||
// dataLen = varDataTLen(pData + sizeof(char));
|
||||
// } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
|
||||
// dataLen = sizeof(int64_t);
|
||||
// } else if (*pData == TSDB_DATA_TYPE_BOOL) {
|
||||
// dataLen = sizeof(char);
|
||||
// }
|
||||
// dataLen += sizeof(char);
|
||||
// }
|
||||
|
||||
if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) {
|
||||
uint32_t newSize = data->varLenCol.payloadAllocLen;
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ typedef struct SDatabaseOptions {
|
|||
int8_t s3Compact;
|
||||
int8_t withArbitrator;
|
||||
// for auto-compact
|
||||
int8_t compactTimeOffset; // hours
|
||||
int32_t compactTimeOffset; // hours
|
||||
int32_t compactInterval; // minutes
|
||||
int32_t compactStartTime; // minutes
|
||||
int32_t compactEndTime; // minutes
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ void streamStateClose(SStreamState* pState, bool remove);
|
|||
int32_t streamStateBegin(SStreamState* pState);
|
||||
void streamStateCommit(SStreamState* pState);
|
||||
void streamStateDestroy(SStreamState* pState, bool remove);
|
||||
int32_t streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
|
||||
int32_t streamStateDelTaskDb(SStreamState* pState);
|
||||
|
||||
int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
|
||||
|
|
@ -108,7 +107,6 @@ int32_t streamStateFillGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, con
|
|||
int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen);
|
||||
|
||||
// twa
|
||||
void streamStateSetFillInfo(SStreamState* pState);
|
||||
void streamStateClearExpiredState(SStreamState* pState);
|
||||
|
||||
void streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur);
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState);
|
|||
void flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState);
|
||||
int32_t recoverSnapshot(SStreamFileState* pFileState, int64_t ckId);
|
||||
|
||||
int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list);
|
||||
int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark);
|
||||
int32_t streamFileStateGetSelectRowSize(SStreamFileState* pFileState);
|
||||
void streamFileStateReloadInfo(SStreamFileState* pFileState, TSKEY ts);
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ int32_t tfsMkdirRecur(STfs *pTfs, const char *rname);
|
|||
* @return int32_t 0 for success, -1 for failure.
|
||||
*/
|
||||
int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId);
|
||||
|
||||
#if 0
|
||||
/**
|
||||
* @brief check directories exist in tfs.
|
||||
*
|
||||
|
|
@ -158,7 +158,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId);
|
|||
* @return true for exist, false for not exist.
|
||||
*/
|
||||
bool tfsDirExistAt(STfs *pTfs, const char *rname, SDiskID diskId);
|
||||
|
||||
#endif
|
||||
/**
|
||||
* @brief Remove directory at all levels in tfs.
|
||||
*
|
||||
|
|
@ -241,7 +241,7 @@ void tfsBasename(const STfsFile *pFile, char *dest);
|
|||
* @param dest The buffer where dirname will be saved.
|
||||
*/
|
||||
void tfsDirname(const STfsFile *pFile, char *dest);
|
||||
|
||||
#if 0
|
||||
/**
|
||||
* @brief Get the absolute file name of rname.
|
||||
*
|
||||
|
|
@ -251,7 +251,7 @@ void tfsDirname(const STfsFile *pFile, char *dest);
|
|||
* @param aname absolute file name
|
||||
*/
|
||||
void tfsAbsoluteName(STfs *pTfs, SDiskID diskId, const char *rname, char *aname);
|
||||
|
||||
#endif
|
||||
/**
|
||||
* @brief Remove file in tfs.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -848,6 +848,7 @@ static int stmtSetDbName2(TAOS_STMT2* stmt, const char* dbName) {
|
|||
STMT_DLOG("start to set dbName: %s", dbName);
|
||||
|
||||
pStmt->db = taosStrdup(dbName);
|
||||
(void)strdequote(pStmt->db);
|
||||
STMT_ERR_RET(stmtCreateRequest(pStmt));
|
||||
|
||||
// The SQL statement specifies a database name, overriding the previously specified database
|
||||
|
|
|
|||
|
|
@ -126,6 +126,10 @@ void fetchCallback(void* param, void* res, int32_t numOfRow) {
|
|||
void queryCallback(void* param, void* res, int32_t code) {
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
(void)printf("failed to execute, reason:%s\n", taos_errstr(res));
|
||||
taos_free_result(res);
|
||||
tsem_t *sem = (tsem_t *)param;
|
||||
tsem_post(sem);
|
||||
return;
|
||||
}
|
||||
(void)printf("start to fetch data\n");
|
||||
taos_fetch_raw_block_a(res, fetchCallback, param);
|
||||
|
|
|
|||
|
|
@ -276,6 +276,7 @@ TEST(testCase, smlParseCols_Test) {
|
|||
info->dataFormat = false;
|
||||
SSmlLineInfo elements = {0};
|
||||
info->msgBuf = msgBuf;
|
||||
ASSERT_EQ(smlInitHandle(NULL), TSDB_CODE_INVALID_PARA);
|
||||
|
||||
const char *data =
|
||||
"st,t=1 cb\\=in=\"pass\\,it "
|
||||
|
|
|
|||
|
|
@ -66,9 +66,9 @@ int32_t s3Begin() {
|
|||
void s3End() { S3_deinitialize(); }
|
||||
|
||||
int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ }
|
||||
|
||||
#if 0
|
||||
static int32_t s3ListBucket(char const *bucketname);
|
||||
|
||||
#endif
|
||||
static void s3DumpCfgByEp(int8_t epIndex) {
|
||||
// clang-format off
|
||||
(void)fprintf(stdout,
|
||||
|
|
@ -291,7 +291,7 @@ static int32_t s3ListBucketByEp(char const *bucketname, int8_t epIndex) {
|
|||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int32_t s3ListBucket(char const *bucketname) {
|
||||
int32_t code = 0;
|
||||
|
||||
|
|
@ -312,7 +312,7 @@ static int32_t s3ListBucket(char const *bucketname) {
|
|||
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
#endif
|
||||
typedef struct growbuffer {
|
||||
// The total number of bytes, and the start byte
|
||||
int size;
|
||||
|
|
|
|||
|
|
@ -11141,6 +11141,7 @@ void tOffsetCopy(STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
|
|||
}
|
||||
|
||||
void tOffsetDestroy(void *param) {
|
||||
if (param == NULL) return;
|
||||
STqOffsetVal *pVal = (STqOffsetVal *)param;
|
||||
if (IS_VAR_DATA_TYPE(pVal->primaryKey.type)) {
|
||||
taosMemoryFreeClear(pVal->primaryKey.pData);
|
||||
|
|
@ -11148,6 +11149,7 @@ void tOffsetDestroy(void *param) {
|
|||
}
|
||||
|
||||
void tDeleteSTqOffset(void *param) {
|
||||
if (param == NULL) return;
|
||||
STqOffset *pVal = (STqOffset *)param;
|
||||
tOffsetDestroy(&pVal->val);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,6 +123,9 @@ static const SSysDbTableSchema userDBSchema[] = {
|
|||
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
|
||||
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
|
||||
{.name = "encrypt_algorithm", .bytes = TSDB_ENCRYPT_ALGO_STR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "compact_interval", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "compact_time_range", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
{.name = "compact_time_offset", .bytes = 4 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema userFuncSchema[] = {
|
||||
|
|
|
|||
|
|
@ -2671,8 +2671,13 @@ static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal
|
|||
tColDataGetValue6, // HAS_VALUE | HAS_NULL
|
||||
tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE
|
||||
};
|
||||
void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
|
||||
int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
|
||||
if (iVal < 0 || iVal >= pColData->nVal ||
|
||||
(pColData->flag <= 0 || pColData->flag >= sizeof(tColDataGetValueImpl)/POINTER_BYTES)){
|
||||
return TSDB_CODE_INVALID_PARA;
|
||||
}
|
||||
tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) {
|
||||
|
|
@ -3436,7 +3441,10 @@ static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, S
|
|||
|
||||
for (int32_t i = 0; i < nColData; i++) {
|
||||
SColVal cv = {0};
|
||||
tColDataGetValue(&aFromColData[i], iFromRow, &cv);
|
||||
code = tColDataGetValue(&aFromColData[i], iFromRow, &cv);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
}
|
||||
code = tColDataAppendValue(&aToColData[i], &cv);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
|
|
@ -3575,7 +3583,10 @@ static int32_t tColDataMerge(SArray **colArr) {
|
|||
SColData *dstCol = taosArrayGet(dst, j);
|
||||
|
||||
SColVal cv;
|
||||
tColDataGetValue(srcCol, i, &cv);
|
||||
code = tColDataGetValue(srcCol, i, &cv);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _exit;
|
||||
}
|
||||
code = tColDataAppendValue(dstCol, &cv);
|
||||
if (code) {
|
||||
goto _exit;
|
||||
|
|
@ -3588,7 +3599,10 @@ static int32_t tColDataMerge(SArray **colArr) {
|
|||
SColData *dstCol = taosArrayGet(dst, j);
|
||||
|
||||
SColVal cv;
|
||||
tColDataGetValue(srcCol, i, &cv);
|
||||
code = tColDataGetValue(srcCol, i, &cv);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
goto _exit;
|
||||
}
|
||||
code = tColDataUpdateValue(dstCol, &cv, true);
|
||||
if (code) {
|
||||
goto _exit;
|
||||
|
|
|
|||
|
|
@ -102,7 +102,6 @@ int32_t tsMaxStreamBackendCache = 128; // M
|
|||
int32_t tsPQSortMemThreshold = 16; // M
|
||||
int32_t tsRetentionSpeedLimitMB = 0; // unlimited
|
||||
|
||||
const char *tsAlterCompactTaskKeywords = "max_compact_tasks";
|
||||
int32_t tsNumOfCompactThreads = 2;
|
||||
int32_t tsNumOfRetentionThreads = 1;
|
||||
|
||||
|
|
@ -327,7 +326,6 @@ char tsUdfdLdLibPath[512] = "";
|
|||
bool tsDisableStream = false;
|
||||
int64_t tsStreamBufferSize = 128 * 1024 * 1024;
|
||||
bool tsFilterScalarMode = false;
|
||||
int tsResolveFQDNRetryTime = 100; // seconds
|
||||
int tsStreamAggCnt = 100000;
|
||||
bool tsStreamCoverage = false;
|
||||
|
||||
|
|
@ -746,7 +744,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX,
|
||||
CFG_SCOPE_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
|
||||
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(
|
||||
cfgAddBool(pCfg, "streamCoverage", tsStreamCoverage, CFG_DYN_CLIENT, CFG_DYN_CLIENT, CFG_CATEGORY_LOCAL));
|
||||
|
||||
TAOS_RETURN(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
|
@ -795,9 +794,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
tsNumOfCommitThreads = tsNumOfCores / 2;
|
||||
tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4);
|
||||
|
||||
tsNumOfCompactThreads = tsNumOfCommitThreads;
|
||||
tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4);
|
||||
|
||||
tsNumOfSupportVnodes = tsNumOfCores * 2 + 5;
|
||||
tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
|
||||
|
||||
|
|
@ -842,7 +838,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY, CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxCompactConcurrency", tsNumOfCompactThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfCompactThreads", tsNumOfCompactThreads, 1, 16, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "queryUseMemoryPool", tsQueryUseMemoryPool, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0);
|
||||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "memPoolFullFunc", tsMemPoolFullFunc, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL) != 0);
|
||||
|
|
@ -956,7 +952,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_RETURN(cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL));
|
||||
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
|
||||
|
||||
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
|
||||
TAOS_CHECK_RETURN(cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint[0], CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER_LAZY,CFG_CATEGORY_GLOBAL));
|
||||
|
|
@ -1040,10 +1035,8 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) {
|
|||
pItem->stype = stype;
|
||||
}
|
||||
|
||||
pItem = cfgGetItem(pCfg, "maxCompactConcurrency");
|
||||
pItem = cfgGetItem(pCfg, "numOfCompactThreads");
|
||||
if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) {
|
||||
tsNumOfCompactThreads = numOfCores / 2;
|
||||
tsNumOfCompactThreads = TRANGE(tsNumOfCompactThreads, 2, 4);
|
||||
pItem->i32 = tsNumOfCompactThreads;
|
||||
pItem->stype = stype;
|
||||
}
|
||||
|
|
@ -1548,7 +1541,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCommitThreads");
|
||||
tsNumOfCommitThreads = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "maxCompactConcurrency");
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "numOfCompactThreads");
|
||||
tsNumOfCompactThreads = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "retentionSpeedLimitMB");
|
||||
|
|
@ -1822,9 +1815,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
|
|||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "pqSortMemThreshold");
|
||||
tsPQSortMemThreshold = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "resolveFQDNRetryTime");
|
||||
tsResolveFQDNRetryTime = pItem->i32;
|
||||
|
||||
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minDiskFreeSize");
|
||||
tsMinDiskFreeSize = pItem->i64;
|
||||
|
||||
|
|
@ -2354,6 +2344,8 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
|
|||
TAOS_RETURN(code);
|
||||
}
|
||||
|
||||
extern void tsdbAlterNumCompactThreads();
|
||||
|
||||
static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = -1;
|
||||
|
|
@ -2404,6 +2396,17 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
|
|||
goto _exit;
|
||||
}
|
||||
|
||||
if (strcasecmp(name, "numOfCompactThreads") == 0) {
|
||||
#ifdef TD_ENTERPRISE
|
||||
tsNumOfCompactThreads = pItem->i32;
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
// tsdbAlterNumCompactThreads();
|
||||
#else
|
||||
code = TSDB_CODE_INVALID_CFG;
|
||||
#endif
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
{ // 'bool/int32_t/int64_t/float/double' variables with general modification function
|
||||
static OptionNameAndVar debugOptions[] = {
|
||||
{"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag},
|
||||
|
|
@ -2453,7 +2456,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, const char *name) {
|
|||
{"randErrorDivisor", &tsRandErrDivisor},
|
||||
{"randErrorScope", &tsRandErrScope},
|
||||
{"syncLogBufferMemoryAllowed", &tsLogBufferMemoryAllowed},
|
||||
{"resolveFQDNRetryTime", &tsResolveFQDNRetryTime},
|
||||
{"syncHeartbeatInterval", &tsHeartbeatInterval},
|
||||
{"syncHeartbeatTimeout", &tsHeartbeatTimeout},
|
||||
{"syncSnapReplMaxWaitN", &tsSnapReplMaxWaitN},
|
||||
|
|
|
|||
|
|
@ -40,6 +40,46 @@ add_test(
|
|||
COMMAND dataformatTest
|
||||
)
|
||||
|
||||
# cosCpTest.cpp
|
||||
add_executable(cosCpTest "")
|
||||
target_sources(
|
||||
cosCpTest
|
||||
PRIVATE
|
||||
"cosCpTest.cpp"
|
||||
)
|
||||
target_link_libraries(cosCpTest gtest gtest_main util common)
|
||||
target_include_directories(
|
||||
cosCpTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/common"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/util"
|
||||
)
|
||||
add_test(
|
||||
NAME cosCpTest
|
||||
COMMAND cosCpTest
|
||||
)
|
||||
|
||||
if(TD_LINUX)
|
||||
|
||||
# cosTest.cpp
|
||||
add_executable(cosTest "")
|
||||
target_sources(
|
||||
cosTest
|
||||
PRIVATE
|
||||
"cosTest.cpp"
|
||||
)
|
||||
target_link_libraries(cosTest gtest gtest_main util common)
|
||||
target_include_directories(
|
||||
cosTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/common"
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/util"
|
||||
)
|
||||
add_test(
|
||||
NAME cosTest
|
||||
COMMAND cosTest
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
if (${TD_LINUX})
|
||||
# tmsg test
|
||||
add_executable(tmsgTest "")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,305 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cos_cp.h>
|
||||
#include <taoserror.h>
|
||||
#include <tglobal.h>
|
||||
#include <iostream>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wwrite-strings"
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
||||
TEST(testCase, cpOpenCloseRemove) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int64_t contentLength = 1024;
|
||||
const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M
|
||||
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
|
||||
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
const int max_part_num = 10000;
|
||||
if (totalSeq > max_part_num) {
|
||||
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
|
||||
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
SCheckpoint cp;
|
||||
char const *file = "./afile";
|
||||
char file_cp_path[TSDB_FILENAME_LEN];
|
||||
|
||||
(void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
|
||||
|
||||
cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
|
||||
if (!cp.parts) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
if (cp.thefile) {
|
||||
EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
if (cp.parts) {
|
||||
taosMemoryFree(cp.parts);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS);
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
||||
TEST(testCase, cpBuild) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int64_t contentLength = 1024;
|
||||
const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M
|
||||
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
|
||||
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
const int max_part_num = 10000;
|
||||
if (totalSeq > max_part_num) {
|
||||
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
|
||||
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
SCheckpoint cp;
|
||||
char const *file = "./afile";
|
||||
char file_cp_path[TSDB_FILENAME_LEN];
|
||||
int64_t lmtime = 20241220141705;
|
||||
char const *upload_id = "upload-id-xxx";
|
||||
|
||||
(void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
|
||||
(void)memset(&cp, 0, sizeof(cp));
|
||||
|
||||
cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
|
||||
if (!cp.parts) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
cos_cp_build_upload(&cp, file, contentLength, lmtime, upload_id, chunk_size);
|
||||
|
||||
EXPECT_EQ(cos_cp_dump(&cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
if (cp.thefile) {
|
||||
EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
if (cp.parts) {
|
||||
taosMemoryFree(cp.parts);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
||||
TEST(testCase, cpLoad) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int64_t contentLength = 1024;
|
||||
const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M
|
||||
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
|
||||
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
const int max_part_num = 10000;
|
||||
if (totalSeq > max_part_num) {
|
||||
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
|
||||
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
SCheckpoint cp;
|
||||
char const *file = "./afile";
|
||||
char file_cp_path[TSDB_FILENAME_LEN];
|
||||
int64_t lmtime = 20241220141705;
|
||||
char const *upload_id = "upload-id-xxx";
|
||||
|
||||
(void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
|
||||
(void)memset(&cp, 0, sizeof(cp));
|
||||
|
||||
cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
|
||||
if (!cp.parts) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
if (taosCheckExistFile(file_cp_path)) {
|
||||
EXPECT_EQ(cos_cp_load(file_cp_path, &cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
EXPECT_EQ(cos_cp_is_valid_upload(&cp, contentLength, lmtime), true);
|
||||
|
||||
EXPECT_EQ(cp.cp_type, COS_CP_TYPE_UPLOAD);
|
||||
EXPECT_EQ(cp.md5, std::string(""));
|
||||
EXPECT_EQ(cp.thefile, nullptr);
|
||||
EXPECT_EQ(std::string(cp.file_path), "./afile");
|
||||
EXPECT_EQ(cp.file_size, 1024);
|
||||
EXPECT_EQ(cp.file_last_modified, 20241220141705);
|
||||
EXPECT_EQ(cp.file_md5, std::string(""));
|
||||
EXPECT_EQ(cp.object_name, std::string(""));
|
||||
EXPECT_EQ(cp.object_size, 0);
|
||||
EXPECT_EQ(cp.object_last_modified, std::string(""));
|
||||
EXPECT_EQ(cp.object_etag, std::string(""));
|
||||
EXPECT_EQ(cp.upload_id, std::string("upload-id-xxx"));
|
||||
|
||||
EXPECT_EQ(cp.part_num, 1);
|
||||
EXPECT_EQ(cp.part_size, 8388608);
|
||||
EXPECT_EQ(cp.parts[0].index, 0);
|
||||
EXPECT_EQ(cp.parts[0].offset, 0);
|
||||
EXPECT_EQ(cp.parts[0].size, 1024);
|
||||
EXPECT_EQ(cp.parts[0].completed, 0);
|
||||
EXPECT_EQ(cp.parts[0].etag, std::string(""));
|
||||
EXPECT_EQ(cp.parts[0].crc64, 0);
|
||||
}
|
||||
|
||||
if (cp.thefile) {
|
||||
EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
if (cp.parts) {
|
||||
taosMemoryFree(cp.parts);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS);
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
||||
TEST(testCase, cpBuildUpdate) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int64_t contentLength = 1024;
|
||||
const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M
|
||||
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
|
||||
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
const int max_part_num = 10000;
|
||||
if (totalSeq > max_part_num) {
|
||||
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
|
||||
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
SCheckpoint cp;
|
||||
char const *file = "./afile";
|
||||
char file_cp_path[TSDB_FILENAME_LEN];
|
||||
int64_t lmtime = 20241220141705;
|
||||
char const *upload_id = "upload-id-xxx";
|
||||
int seq = 1;
|
||||
char *etags[1] = {"etags-1-xxx"};
|
||||
|
||||
(void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
|
||||
(void)memset(&cp, 0, sizeof(cp));
|
||||
|
||||
cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
|
||||
if (!cp.parts) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_open(file_cp_path, &cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
cos_cp_build_upload(&cp, file, contentLength, lmtime, upload_id, chunk_size);
|
||||
|
||||
cos_cp_update(&cp, cp.parts[seq - 1].index, etags[seq - 1], 0);
|
||||
|
||||
EXPECT_EQ(cos_cp_dump(&cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
if (cp.thefile) {
|
||||
EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
if (cp.parts) {
|
||||
taosMemoryFree(cp.parts);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
||||
TEST(testCase, cpLoadUpdate) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int64_t contentLength = 1024;
|
||||
const int64_t MULTIPART_CHUNK_SIZE = 64 << 20; // multipart is 768M
|
||||
uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3;
|
||||
int totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
const int max_part_num = 10000;
|
||||
if (totalSeq > max_part_num) {
|
||||
chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
|
||||
totalSeq = (contentLength + chunk_size - 1) / chunk_size;
|
||||
}
|
||||
SCheckpoint cp;
|
||||
char const *file = "./afile";
|
||||
char file_cp_path[TSDB_FILENAME_LEN];
|
||||
int64_t lmtime = 20241220141705;
|
||||
char const *upload_id = "upload-id-xxx";
|
||||
|
||||
(void)snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
|
||||
(void)memset(&cp, 0, sizeof(cp));
|
||||
|
||||
cp.parts = (SCheckpointPart *)taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
|
||||
if (!cp.parts) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
if (taosCheckExistFile(file_cp_path)) {
|
||||
EXPECT_EQ(cos_cp_load(file_cp_path, &cp), TSDB_CODE_SUCCESS);
|
||||
|
||||
EXPECT_EQ(cos_cp_is_valid_upload(&cp, contentLength, lmtime), true);
|
||||
|
||||
EXPECT_EQ(cp.cp_type, COS_CP_TYPE_UPLOAD);
|
||||
EXPECT_EQ(cp.md5, std::string(""));
|
||||
EXPECT_EQ(cp.thefile, nullptr);
|
||||
EXPECT_EQ(std::string(cp.file_path), "./afile");
|
||||
EXPECT_EQ(cp.file_size, 1024);
|
||||
EXPECT_EQ(cp.file_last_modified, 20241220141705);
|
||||
EXPECT_EQ(cp.file_md5, std::string(""));
|
||||
EXPECT_EQ(cp.object_name, std::string(""));
|
||||
EXPECT_EQ(cp.object_size, 0);
|
||||
EXPECT_EQ(cp.object_last_modified, std::string(""));
|
||||
EXPECT_EQ(cp.object_etag, std::string(""));
|
||||
EXPECT_EQ(cp.upload_id, std::string("upload-id-xxx"));
|
||||
|
||||
EXPECT_EQ(cp.part_num, 1);
|
||||
EXPECT_EQ(cp.part_size, 8388608);
|
||||
EXPECT_EQ(cp.parts[0].index, 0);
|
||||
EXPECT_EQ(cp.parts[0].offset, 0);
|
||||
EXPECT_EQ(cp.parts[0].size, 1024);
|
||||
EXPECT_EQ(cp.parts[0].completed, 1);
|
||||
EXPECT_EQ(cp.parts[0].etag, std::string("etags-1-xxx"));
|
||||
EXPECT_EQ(cp.parts[0].crc64, 0);
|
||||
}
|
||||
|
||||
if (cp.thefile) {
|
||||
EXPECT_EQ(cos_cp_close(cp.thefile), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
if (cp.parts) {
|
||||
taosMemoryFree(cp.parts);
|
||||
}
|
||||
|
||||
EXPECT_EQ(cos_cp_remove(file_cp_path), TSDB_CODE_SUCCESS);
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
|
@ -0,0 +1,185 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <cos.h>
|
||||
#include <taoserror.h>
|
||||
#include <tglobal.h>
|
||||
#include <iostream>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wwrite-strings"
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
||||
int32_t cosInitEnv() {
|
||||
int32_t code = 0;
|
||||
bool isBlob = false;
|
||||
|
||||
extern int8_t tsS3Ablob;
|
||||
extern char tsS3Hostname[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
|
||||
extern char tsS3BucketName[TSDB_FQDN_LEN];
|
||||
|
||||
tsS3Ablob = isBlob;
|
||||
/*
|
||||
const char *hostname = "endpoint/<account-name>.blob.core.windows.net";
|
||||
const char *accessKeyId = "<access-key-id/account-name>";
|
||||
const char *accessKeySecret = "<access-key-secret/account-key>";
|
||||
const char *bucketName = "<bucket/container-name>";
|
||||
*/
|
||||
|
||||
// const char *hostname = "http://192.168.1.52:9000";
|
||||
// const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel";
|
||||
// const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX";
|
||||
// const char *bucketName = "test-bucket";
|
||||
const char *hostname = "192.168.1.52:9000";
|
||||
const char *accessKeyId = "zOgllR6bSnw2Ah3mCNel";
|
||||
const char *accessKeySecret = "cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX";
|
||||
const char *bucketName = "ci-bucket19";
|
||||
|
||||
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
|
||||
|
||||
// setup s3 env
|
||||
extern int8_t tsS3EpNum;
|
||||
extern int8_t tsS3Https[TSDB_MAX_EP_NUM];
|
||||
|
||||
tsS3EpNum = 1;
|
||||
tsS3Https[0] = false;
|
||||
|
||||
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
|
||||
|
||||
tsS3Enabled = true;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
TEST(testCase, cosCpPutError) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
char const *objectName = "testObject";
|
||||
|
||||
EXPECT_EQ(cosInitEnv(), TSDB_CODE_SUCCESS);
|
||||
EXPECT_EQ(s3Begin(), TSDB_CODE_SUCCESS);
|
||||
|
||||
#if defined(USE_S3)
|
||||
EXPECT_EQ(s3Size(objectName), -1);
|
||||
#else
|
||||
EXPECT_EQ(s3Size(objectName), 0);
|
||||
#endif
|
||||
|
||||
s3EvictCache("", 0);
|
||||
|
||||
s3End();
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
||||
TEST(testCase, cosCpPut) {
|
||||
int32_t code = 0, lino = 0;
|
||||
|
||||
int8_t with_cp = 0;
|
||||
char *data = nullptr;
|
||||
|
||||
const long objectSize = 65 * 1024 * 1024;
|
||||
char const *objectName = "cosut.bin";
|
||||
const char object_name[] = "cosut.bin";
|
||||
|
||||
EXPECT_EQ(std::string(object_name), objectName);
|
||||
|
||||
EXPECT_EQ(cosInitEnv(), TSDB_CODE_SUCCESS);
|
||||
EXPECT_EQ(s3Begin(), TSDB_CODE_SUCCESS);
|
||||
|
||||
{
|
||||
data = (char *)taosMemoryCalloc(1, objectSize);
|
||||
if (!data) {
|
||||
TAOS_CHECK_EXIT(terrno);
|
||||
}
|
||||
|
||||
for (int i = 0; i < objectSize / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, objectSize);
|
||||
GTEST_ASSERT_EQ(n, objectSize);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = s3PutObjectFromFile2(path, objectName, with_cp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
with_cp = 1;
|
||||
code = s3PutObjectFromFile2(path, objectName, with_cp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
#if defined(USE_S3)
|
||||
EXPECT_EQ(s3Size(objectName), objectSize);
|
||||
#else
|
||||
EXPECT_EQ(s3Size(objectName), 0);
|
||||
#endif
|
||||
|
||||
s3End();
|
||||
s3EvictCache("", 0);
|
||||
|
||||
taosMemoryFree(data);
|
||||
|
||||
EXPECT_EQ(taosRemoveFile(path), TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
_exit:
|
||||
if (data) {
|
||||
taosMemoryFree(data);
|
||||
s3End();
|
||||
}
|
||||
|
||||
std::cout << "code: " << code << std::endl;
|
||||
}
|
||||
|
|
@ -449,6 +449,20 @@ static void checkTSRow(const char **data, STSRow *row, STSchema *pTSchema) {
|
|||
checkSColVal(data[i], &colVal, pCol->type);
|
||||
}
|
||||
}
|
||||
#ifndef WINDOWS
|
||||
TEST(testCase, tColDataGetValue) {
|
||||
SColData pColData = {0};
|
||||
SColVal pColVal = {0};
|
||||
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
|
||||
|
||||
pColData = {.flag = 8};
|
||||
pColVal = {0};
|
||||
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
|
||||
|
||||
pColData = {.nVal = 1, .flag = 8};
|
||||
ASSERT_NE(tColDataGetValue(&pColData, 0, &pColVal),0);
|
||||
}
|
||||
#endif
|
||||
|
||||
TEST(testCase, AllNormTest) {
|
||||
int16_t nCols = 14;
|
||||
|
|
|
|||
|
|
@ -475,27 +475,6 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
extern void tsdbAlterNumCompactThreads();
|
||||
static int32_t dmAlterMaxCompactTask(const char *value) {
|
||||
int32_t max_compact_tasks;
|
||||
char *endptr = NULL;
|
||||
|
||||
max_compact_tasks = taosStr2Int32(value, &endptr, 10);
|
||||
if (endptr == value || endptr[0] != '\0') {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
|
||||
if (max_compact_tasks != tsNumOfCompactThreads) {
|
||||
dInfo("alter max compact tasks from %d to %d", tsNumOfCompactThreads, max_compact_tasks);
|
||||
tsNumOfCompactThreads = max_compact_tasks;
|
||||
#ifdef TD_ENTERPRISE
|
||||
(void)tsdbAlterNumCompactThreads();
|
||||
#endif
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
||||
int32_t code = 0;
|
||||
SDCfgDnodeReq cfgReq = {0};
|
||||
|
|
@ -509,10 +488,6 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) {
|
|||
return taosUpdateTfsItemDisable(pCfg, cfgReq.value, pMgmt->pTfs);
|
||||
}
|
||||
|
||||
if (strncmp(cfgReq.config, tsAlterCompactTaskKeywords, strlen(tsAlterCompactTaskKeywords) + 1) == 0) {
|
||||
return dmAlterMaxCompactTask(cfgReq.value);
|
||||
}
|
||||
|
||||
dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
|
||||
|
||||
code = cfgGetAndSetItem(pCfg, &pItem, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_SERVER_CMD, true);
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
|||
|
|
@ -348,6 +348,7 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
|
|||
req.compactId = compactId;
|
||||
req.vgId = pVgroup->vgId;
|
||||
req.dnodeId = dnodeid;
|
||||
terrno = 0;
|
||||
|
||||
mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId);
|
||||
int32_t contLen = tSerializeSVKillCompactReq(NULL, 0, &req);
|
||||
|
|
@ -367,8 +368,10 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
|
|||
pHead->contLen = htonl(contLen);
|
||||
pHead->vgId = htonl(pVgroup->vgId);
|
||||
|
||||
if ((contLen = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) {
|
||||
terrno = contLen;
|
||||
mTrace("vgId:%d, build compact vnode config req, contLen:%d", pVgroup->vgId, contLen);
|
||||
int32_t ret = 0;
|
||||
if ((ret = tSerializeSVKillCompactReq((char *)pReq + sizeof(SMsgHead), contLen, &req)) < 0) {
|
||||
terrno = ret;
|
||||
return NULL;
|
||||
}
|
||||
*pContLen = contLen;
|
||||
|
|
@ -401,6 +404,8 @@ static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *p
|
|||
action.contLen = contLen;
|
||||
action.msgType = TDMT_VND_KILL_COMPACT;
|
||||
|
||||
mTrace("trans:%d, kill compact msg len:%d", pTrans->id, contLen);
|
||||
|
||||
if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
|
||||
taosMemoryFree(pReq);
|
||||
TAOS_RETURN(code);
|
||||
|
|
|
|||
|
|
@ -499,6 +499,20 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
|
|||
if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code;
|
||||
if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code;
|
||||
|
||||
if (pCfg->compactInterval != 0 &&
|
||||
(pCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pCfg->compactInterval > pCfg->daysToKeep2))
|
||||
return code;
|
||||
if (pCfg->compactStartTime != 0 &&
|
||||
(pCfg->compactStartTime < -pCfg->daysToKeep2 || pCfg->compactStartTime > -pCfg->daysPerFile))
|
||||
return code;
|
||||
if (pCfg->compactEndTime != 0 &&
|
||||
(pCfg->compactEndTime < -pCfg->daysToKeep2 || pCfg->compactEndTime > -pCfg->daysPerFile))
|
||||
return code;
|
||||
if (pCfg->compactStartTime != 0 && pCfg->compactEndTime != 0 && pCfg->compactStartTime >= pCfg->compactEndTime)
|
||||
return code;
|
||||
if (pCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET || pCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET)
|
||||
return code;
|
||||
|
||||
code = 0;
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
|
@ -564,6 +578,22 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
|
|||
if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return code;
|
||||
if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return code;
|
||||
|
||||
if (pNewCfg->compactInterval != 0 &&
|
||||
(pNewCfg->compactInterval < TSDB_MIN_COMPACT_INTERVAL || pNewCfg->compactInterval > pNewCfg->daysToKeep2))
|
||||
return code;
|
||||
if (pNewCfg->compactStartTime != 0 &&
|
||||
(pNewCfg->compactStartTime < -pNewCfg->daysToKeep2 || pNewCfg->compactStartTime > -pNewCfg->daysPerFile))
|
||||
return code;
|
||||
if (pNewCfg->compactEndTime != 0 &&
|
||||
(pNewCfg->compactEndTime < -pNewCfg->daysToKeep2 || pNewCfg->compactEndTime > -pNewCfg->daysPerFile))
|
||||
return code;
|
||||
if (pNewCfg->compactStartTime != 0 && pNewCfg->compactEndTime != 0 &&
|
||||
pNewCfg->compactStartTime >= pNewCfg->compactEndTime)
|
||||
return code;
|
||||
if (pNewCfg->compactTimeOffset < TSDB_MIN_COMPACT_TIME_OFFSET ||
|
||||
pNewCfg->compactTimeOffset > TSDB_MAX_COMPACT_TIME_OFFSET)
|
||||
return code;
|
||||
|
||||
code = 0;
|
||||
TAOS_RETURN(code);
|
||||
}
|
||||
|
|
@ -1150,20 +1180,24 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
|
|||
code = 0;
|
||||
}
|
||||
|
||||
bool compactTimeRangeChanged = false;
|
||||
if (pAlter->compactStartTime != pDb->cfg.compactStartTime &&
|
||||
(pAlter->compactStartTime == TSDB_DEFAULT_COMPACT_START_TIME ||
|
||||
pAlter->compactStartTime <= -pDb->cfg.daysPerFile)) {
|
||||
pDb->cfg.compactStartTime = pAlter->compactStartTime;
|
||||
pDb->vgVersion++;
|
||||
compactTimeRangeChanged = true;
|
||||
code = 0;
|
||||
}
|
||||
|
||||
if (pAlter->compactEndTime != pDb->cfg.compactEndTime &&
|
||||
(pAlter->compactEndTime == TSDB_DEFAULT_COMPACT_END_TIME || pAlter->compactEndTime <= -pDb->cfg.daysPerFile)) {
|
||||
pDb->cfg.compactEndTime = pAlter->compactEndTime;
|
||||
pDb->vgVersion++;
|
||||
compactTimeRangeChanged = true;
|
||||
code = 0;
|
||||
}
|
||||
if(compactTimeRangeChanged) {
|
||||
pDb->vgVersion++;
|
||||
}
|
||||
|
||||
if (pAlter->compactTimeOffset >= TSDB_MIN_COMPACT_TIME_OFFSET &&
|
||||
pAlter->compactTimeOffset != pDb->cfg.compactTimeOffset) {
|
||||
|
|
@ -1408,14 +1442,6 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
|
|||
cfgRsp->compactInterval = pDb->cfg.compactInterval;
|
||||
cfgRsp->compactStartTime = pDb->cfg.compactStartTime;
|
||||
cfgRsp->compactEndTime = pDb->cfg.compactEndTime;
|
||||
if (cfgRsp->compactInterval > 0) {
|
||||
if (cfgRsp->compactStartTime == 0) {
|
||||
cfgRsp->compactStartTime = -cfgRsp->daysToKeep2;
|
||||
}
|
||||
if (cfgRsp->compactEndTime == 0) {
|
||||
cfgRsp->compactEndTime = -cfgRsp->daysPerFile;
|
||||
}
|
||||
}
|
||||
cfgRsp->compactTimeOffset = pDb->cfg.compactTimeOffset;
|
||||
}
|
||||
|
||||
|
|
@ -2432,6 +2458,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
|
|||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)strictVstr, false), &lino, _OVER);
|
||||
|
||||
char durationStr[128] = {0};
|
||||
char durationVstr[128] = {0};
|
||||
int32_t len = formatDurationOrKeep(&durationVstr[VARSTR_HEADER_SIZE], sizeof(durationVstr) - VARSTR_HEADER_SIZE,
|
||||
pDb->cfg.daysPerFile);
|
||||
|
|
@ -2440,10 +2467,10 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
|
|||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER);
|
||||
|
||||
char keepVstr[512] = {0};
|
||||
char keep0Str[128] = {0};
|
||||
char keep1Str[128] = {0};
|
||||
char keep2Str[128] = {0};
|
||||
char keepVstr[128] = {0};
|
||||
char keep0Str[32] = {0};
|
||||
char keep1Str[32] = {0};
|
||||
char keep2Str[32] = {0};
|
||||
|
||||
int32_t lenKeep0 = formatDurationOrKeep(keep0Str, sizeof(keep0Str), pDb->cfg.daysToKeep0);
|
||||
int32_t lenKeep1 = formatDurationOrKeep(keep1Str, sizeof(keep1Str), pDb->cfg.daysToKeep1);
|
||||
|
|
@ -2556,6 +2583,26 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
|
|||
STR_WITH_MAXSIZE_TO_VARSTR(encryptAlgorithmVStr, encryptAlgorithmStr, 24);
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)encryptAlgorithmVStr, false), &lino, _OVER);
|
||||
|
||||
TAOS_UNUSED(formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactInterval));
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr));
|
||||
if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) {
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER);
|
||||
}
|
||||
|
||||
len = formatDurationOrKeep(durationStr, sizeof(durationStr), pDb->cfg.compactStartTime);
|
||||
TAOS_UNUSED(formatDurationOrKeep(durationVstr, sizeof(durationVstr), pDb->cfg.compactEndTime));
|
||||
TAOS_UNUSED(snprintf(durationStr + len, sizeof(durationStr) - len, ",%s", durationVstr));
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr));
|
||||
if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) {
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER);
|
||||
}
|
||||
|
||||
TAOS_UNUSED(snprintf(durationStr, sizeof(durationStr), "%dh", pDb->cfg.compactTimeOffset));
|
||||
STR_WITH_MAXSIZE_TO_VARSTR(durationVstr, durationStr, sizeof(durationVstr));
|
||||
if ((pColInfo = taosArrayGet(pBlock->pDataBlock, cols++))) {
|
||||
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)durationVstr, false), &lino, _OVER);
|
||||
}
|
||||
}
|
||||
_OVER:
|
||||
if (code != 0) mError("failed to retrieve at line:%d, since %s", lino, tstrerror(code));
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur;
|
||||
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
|
||||
|
||||
pStore->streamStateSetFillInfo = streamStateSetFillInfo;
|
||||
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
|
||||
|
||||
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
|
||||
|
|
@ -117,7 +116,6 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->streamStateBegin = streamStateBegin;
|
||||
pStore->streamStateCommit = streamStateCommit;
|
||||
pStore->streamStateDestroy = streamStateDestroy;
|
||||
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
||||
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
||||
pStore->streamStateCopyBackend = streamStateCopyBackend;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ int32_t metaAlterSuperTable(SMeta* pMeta, int64_t version, SVCreateStbRe
|
|||
int32_t metaDropSuperTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
|
||||
int32_t metaCreateTable2(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** ppRsp);
|
||||
int32_t metaDropTable2(SMeta* pMeta, int64_t version, SVDropTbReq* pReq);
|
||||
int32_t metaTrimTables(SMeta* pMeta);
|
||||
int32_t metaTrimTables(SMeta* pMeta, int64_t version);
|
||||
int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tbUids);
|
||||
int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount);
|
||||
int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp);
|
||||
|
|
|
|||
|
|
@ -1645,7 +1645,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry
|
|||
metaFetchEntryFree(&pOldEntry);
|
||||
return code;
|
||||
}
|
||||
// TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids));
|
||||
TAOS_CHECK_RETURN(tsdbCacheNewSTableColumn(pTsdb, uids, cid, col_type));
|
||||
} else if (deltaCol == -1) {
|
||||
int16_t cid = -1;
|
||||
|
|
@ -1667,7 +1666,6 @@ static int32_t metaHandleSuperTableUpdate(SMeta *pMeta, const SMetaEntry *pEntry
|
|||
metaFetchEntryFree(&pOldEntry);
|
||||
return code;
|
||||
}
|
||||
// TAOS_CHECK_RETURN(metaGetSubtables(pMeta, pEntry->uid, uids));
|
||||
TAOS_CHECK_RETURN(tsdbCacheDropSTableColumn(pTsdb, uids, cid, hasPrimaryKey));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -325,7 +325,6 @@ int32_t buildSnapContext(SVnode* pVnode, int64_t snapVersion, int64_t suid, int8
|
|||
ctx->suidInfo = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
|
||||
if (ctx->suidInfo == NULL) {
|
||||
return TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
|
||||
;
|
||||
}
|
||||
taosHashSetFreeFp(ctx->suidInfo, destroySTableInfoForChildTable);
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -151,11 +151,8 @@ void tqClose(STQ* pTq) {
|
|||
taosHashCleanup(pTq->pOffset);
|
||||
taosMemoryFree(pTq->path);
|
||||
tqMetaClose(pTq);
|
||||
|
||||
int32_t vgId = pTq->pStreamMeta->vgId;
|
||||
qDebug("vgId:%d end to close tq", pTq->pStreamMeta != NULL ? pTq->pStreamMeta->vgId : -1);
|
||||
streamMetaClose(pTq->pStreamMeta);
|
||||
|
||||
qDebug("vgId:%d end to close tq", vgId);
|
||||
taosMemoryFree(pTq);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,36 +17,41 @@
|
|||
#include "tq.h"
|
||||
|
||||
int32_t tqBuildFName(char** data, const char* path, char* name) {
|
||||
if (data == NULL || path == NULL || name == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
char* fname = NULL;
|
||||
TSDB_CHECK_NULL(data, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(path, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(name, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
int32_t len = strlen(path) + strlen(name) + 2;
|
||||
char* fname = taosMemoryCalloc(1, len);
|
||||
if(fname == NULL) {
|
||||
return terrno;
|
||||
fname = taosMemoryCalloc(1, len);
|
||||
TSDB_CHECK_NULL(fname, code, lino, END, terrno);
|
||||
(void)tsnprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name);
|
||||
|
||||
*data = fname;
|
||||
fname = NULL;
|
||||
|
||||
END:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
int32_t code = tsnprintf(fname, len, "%s%s%s", path, TD_DIRSEP, name);
|
||||
if (code < 0){
|
||||
code = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(fname);
|
||||
return code;
|
||||
}
|
||||
*data = fname;
|
||||
return TDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
|
||||
if (pTq == NULL || name == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = TDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* pMemBuf = NULL;
|
||||
TdFilePtr pFile = NULL;
|
||||
STqOffset *pOffset = NULL;
|
||||
void *pIter = NULL;
|
||||
|
||||
TdFilePtr pFile = taosOpenFile(name, TD_FILE_READ);
|
||||
if (pFile == NULL) {
|
||||
code = TDB_CODE_SUCCESS;
|
||||
goto END;
|
||||
}
|
||||
TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(name, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
pFile = taosOpenFile(name, TD_FILE_READ);
|
||||
TSDB_CHECK_NULL(pFile, code, lino, END, TDB_CODE_SUCCESS);
|
||||
|
||||
int64_t ret = 0;
|
||||
int32_t size = 0;
|
||||
|
|
@ -60,48 +65,41 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name) {
|
|||
}
|
||||
total += INT_BYTES;
|
||||
size = htonl(size);
|
||||
if (size <= 0) {
|
||||
code = TSDB_CODE_INVALID_MSG;
|
||||
goto END;
|
||||
}
|
||||
pMemBuf = taosMemoryCalloc(1, size);
|
||||
if (pMemBuf == NULL) {
|
||||
code = terrno;
|
||||
goto END;
|
||||
}
|
||||
TSDB_CHECK_CONDITION(size > 0, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
if (taosReadFile(pFile, pMemBuf, size) != size) {
|
||||
terrno = TSDB_CODE_INVALID_MSG;
|
||||
goto END;
|
||||
}
|
||||
pMemBuf = taosMemoryCalloc(1, size);
|
||||
TSDB_CHECK_NULL(pMemBuf, code, lino, END, terrno);
|
||||
TSDB_CHECK_CONDITION(taosReadFile(pFile, pMemBuf, size) == size, code, lino, END, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
total += size;
|
||||
STqOffset offset = {0};
|
||||
TQ_ERR_GO_TO_END(tqMetaDecodeOffsetInfo(&offset, pMemBuf, size));
|
||||
code = taosHashPut(pTq->pOffset, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset));
|
||||
if (code != TDB_CODE_SUCCESS) {
|
||||
tDeleteSTqOffset(&offset);
|
||||
goto END;
|
||||
}
|
||||
code = tqMetaDecodeOffsetInfo(&offset, pMemBuf, size);
|
||||
TSDB_CHECK_CODE(code, lino, END);
|
||||
pOffset = &offset;
|
||||
code = taosHashPut(pTq->pOffset, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
|
||||
TSDB_CHECK_CODE(code, lino, END);
|
||||
pOffset = NULL;
|
||||
|
||||
tqInfo("tq: offset restore from file to tdb, size:%d, hash size:%d subkey:%s", total, taosHashGetSize(pTq->pOffset), offset.subKey);
|
||||
taosMemoryFree(pMemBuf);
|
||||
pMemBuf = NULL;
|
||||
}
|
||||
|
||||
void *pIter = NULL;
|
||||
while ((pIter = taosHashIterate(pTq->pOffset, pIter))) {
|
||||
STqOffset* pOffset = (STqOffset*)pIter;
|
||||
code = tqMetaSaveOffset(pTq, pOffset);
|
||||
if(code != 0){
|
||||
taosHashCancelIterate(pTq->pOffset, pIter);
|
||||
goto END;
|
||||
}
|
||||
STqOffset* offset = (STqOffset*)pIter;
|
||||
code = tqMetaSaveOffset(pTq, offset);
|
||||
TSDB_CHECK_CODE(code, lino, END);
|
||||
}
|
||||
|
||||
END:
|
||||
taosCloseFile(&pFile);
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
(void)taosCloseFile(&pFile);
|
||||
taosMemoryFree(pMemBuf);
|
||||
|
||||
tDeleteSTqOffset(pOffset);
|
||||
taosHashCancelIterate(pTq->pOffset, pIter);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -796,7 +796,8 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
|
|||
sourceIdx++;
|
||||
} else if (pCol->cid == pColData->info.colId) {
|
||||
for (int32_t i = 0; i < pCol->nVal; i++) {
|
||||
tColDataGetValue(pCol, i, &colVal);
|
||||
code = tColDataGetValue(pCol, i, &colVal);
|
||||
TSDB_CHECK_CODE(code, line, END);
|
||||
code = doSetVal(pColData, i, &colVal);
|
||||
TSDB_CHECK_CODE(code, line, END);
|
||||
}
|
||||
|
|
@ -937,7 +938,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
|
|||
pCol = taosArrayGet(pCols, j);
|
||||
TQ_NULL_GO_TO_END(pCol);
|
||||
SColVal colVal = {0};
|
||||
tColDataGetValue(pCol, i, &colVal);
|
||||
TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal));
|
||||
PROCESS_VAL
|
||||
}
|
||||
|
||||
|
|
@ -961,7 +962,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData
|
|||
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx);
|
||||
TQ_NULL_GO_TO_END(pColData);
|
||||
SColVal colVal = {0};
|
||||
tColDataGetValue(pCol, i, &colVal);
|
||||
TQ_ERR_GO_TO_END(tColDataGetValue(pCol, i, &colVal));
|
||||
SET_DATA
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,18 +27,16 @@ struct STqSnapReader {
|
|||
};
|
||||
|
||||
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {
|
||||
if (pTq == NULL || ppReader == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
STqSnapReader* pReader = NULL;
|
||||
TSDB_CHECK_NULL(pTq, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(ppReader, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
// alloc
|
||||
pReader = (STqSnapReader*)taosMemoryCalloc(1, sizeof(STqSnapReader));
|
||||
if (pReader == NULL) {
|
||||
code = terrno;
|
||||
goto _err;
|
||||
}
|
||||
TSDB_CHECK_NULL(pReader, code, lino, end, terrno);
|
||||
|
||||
pReader->pTq = pTq;
|
||||
pReader->sver = sver;
|
||||
pReader->ever = ever;
|
||||
|
|
@ -54,28 +52,21 @@ int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqS
|
|||
pTb = pTq->pOffsetStore;
|
||||
} else {
|
||||
code = TSDB_CODE_INVALID_MSG;
|
||||
goto _err;
|
||||
goto end;
|
||||
}
|
||||
code = tdbTbcOpen(pTb, &pReader->pCur, NULL);
|
||||
if (code) {
|
||||
taosMemoryFree(pReader);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
code = tdbTbcMoveToFirst(pReader->pCur);
|
||||
if (code) {
|
||||
taosMemoryFree(pReader);
|
||||
goto _err;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
tqInfo("vgId:%d, vnode tq snapshot reader opene success", TD_VID(pTq->pVnode));
|
||||
*ppReader = pReader;
|
||||
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d, vnode tq snapshot reader open failed since %s", __func__, lino, tstrerror(code));
|
||||
taosMemoryFreeClear(pReader);
|
||||
}
|
||||
|
||||
tqInfo("vgId:%d, vnode snapshot tq reader opened", TD_VID(pTq->pVnode));
|
||||
|
||||
*ppReader = pReader;
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, vnode snapshot tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
*ppReader = NULL;
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -84,45 +75,37 @@ void tqSnapReaderClose(STqSnapReader** ppReader) {
|
|||
return;
|
||||
}
|
||||
tdbTbcClose((*ppReader)->pCur);
|
||||
taosMemoryFree(*ppReader);
|
||||
*ppReader = NULL;
|
||||
taosMemoryFreeClear(*ppReader);
|
||||
}
|
||||
|
||||
int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
|
||||
if (pReader == NULL || ppData == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
void* pKey = NULL;
|
||||
void* pVal = NULL;
|
||||
int32_t kLen = 0;
|
||||
int32_t vLen = 0;
|
||||
TSDB_CHECK_NULL(pReader, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(ppData, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
|
||||
goto _exit;
|
||||
}
|
||||
code = tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen);
|
||||
TSDB_CHECK_CONDITION(code == 0, code, lino, end, TDB_CODE_SUCCESS);
|
||||
|
||||
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
|
||||
if (*ppData == NULL) {
|
||||
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
|
||||
goto _err;
|
||||
}
|
||||
TSDB_CHECK_NULL(*ppData, code, lino, end, terrno);
|
||||
|
||||
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
|
||||
pHdr->type = pReader->type;
|
||||
pHdr->size = vLen;
|
||||
(void)memcpy(pHdr->data, pVal, vLen);
|
||||
tqInfo("vgId:%d, vnode tq snapshot read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
|
||||
|
||||
_exit:
|
||||
end:
|
||||
if (code != 0) {
|
||||
tqError("%s failed at %d, vnode tq snapshot read data failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
tqInfo("vgId:%d, vnode snapshot tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tdbFree(pKey);
|
||||
tdbFree(pVal);
|
||||
tqError("vgId:%d, vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
|
||||
return code;
|
||||
}
|
||||
|
||||
|
|
@ -135,135 +118,148 @@ struct STqSnapWriter {
|
|||
};
|
||||
|
||||
int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {
|
||||
if (pTq == NULL || ppWriter == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
STqSnapWriter* pWriter = NULL;
|
||||
|
||||
TSDB_CHECK_NULL(pTq, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(ppWriter, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
|
||||
// alloc
|
||||
pWriter = (STqSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
|
||||
if (pWriter == NULL) {
|
||||
code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
|
||||
;
|
||||
goto _err;
|
||||
}
|
||||
TSDB_CHECK_NULL(pWriter, code, lino, end, terrno);
|
||||
pWriter->pTq = pTq;
|
||||
pWriter->sver = sver;
|
||||
pWriter->ever = ever;
|
||||
|
||||
code = tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
|
||||
if (code < 0) {
|
||||
taosMemoryFree(pWriter);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
tqInfo("vgId:%d, tq snapshot writer opene success", TD_VID(pTq->pVnode));
|
||||
*ppWriter = pWriter;
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
*ppWriter = NULL;
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d tq snapshot writer open failed since %s", __func__, lino, tstrerror(code));
|
||||
taosMemoryFreeClear(pWriter);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
|
||||
if (ppWriter == NULL || *ppWriter == NULL) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
STqSnapWriter* pWriter = *ppWriter;
|
||||
STQ* pTq = pWriter->pTq;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
STqSnapWriter* pWriter = NULL;
|
||||
|
||||
TSDB_CHECK_NULL(ppWriter, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(*ppWriter, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
pWriter = *ppWriter;
|
||||
|
||||
if (rollback) {
|
||||
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
} else {
|
||||
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
if (code) goto _err;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
|
||||
if (code) goto _err;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
}
|
||||
tqInfo("vgId:%d, tq snapshot writer close success", TD_VID(pWriter->pTq->pVnode));
|
||||
taosMemoryFreeClear(*ppWriter);
|
||||
|
||||
taosMemoryFree(pWriter);
|
||||
*ppWriter = NULL;
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d, tq snapshot writer close failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
static int32_t tqWriteCheck(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData){
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
TSDB_CHECK_NULL(pWriter, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_NULL(pData, code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
TSDB_CHECK_CONDITION(nData >= sizeof(SSnapDataHdr), code, lino, end, TSDB_CODE_INVALID_MSG);
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
|
||||
if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
STQ* pTq = pWriter->pTq;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
SDecoder decoder = {0};
|
||||
SDecoder* pDecoder = &decoder;
|
||||
STqHandle handle = {0};
|
||||
code = tqWriteCheck(pWriter, pData, nData);
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
STQ* pTq = pWriter->pTq;
|
||||
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
code = tDecodeSTqHandle(pDecoder, &handle);
|
||||
if (code) goto end;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
taosWLockLatch(&pTq->lock);
|
||||
code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, strlen(handle.subKey), pData + sizeof(SSnapDataHdr),
|
||||
nData - sizeof(SSnapDataHdr));
|
||||
taosWUnLockLatch(&pTq->lock);
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
tqInfo("vgId:%d, vnode tq snapshot write success", TD_VID(pTq->pVnode));
|
||||
|
||||
end:
|
||||
tDecoderClear(pDecoder);
|
||||
tqDestroyTqHandle(&handle);
|
||||
tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d, vnode tq snapshot write failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
|
||||
if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
|
||||
code = tqWriteCheck(pWriter, pData, nData);
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
STQ* pTq = pWriter->pTq;
|
||||
STqCheckInfo info = {0};
|
||||
code = tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
if (code != 0) {
|
||||
goto _err;
|
||||
}
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr),
|
||||
nData - sizeof(SSnapDataHdr));
|
||||
tDeleteSTqCheckInfo(&info);
|
||||
if (code) goto _err;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
tqInfo("vgId:%d, vnode tq check info write success", TD_VID(pTq->pVnode));
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d, vnode tq check info write failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
|
||||
if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
|
||||
return TSDB_CODE_INVALID_MSG;
|
||||
}
|
||||
int32_t code = 0;
|
||||
STQ* pTq = pWriter->pTq;
|
||||
int code = TSDB_CODE_SUCCESS;
|
||||
int32_t lino = 0;
|
||||
code = tqWriteCheck(pWriter, pData, nData);
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
STQ* pTq = pWriter->pTq;
|
||||
STqOffset info = {0};
|
||||
code = tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
|
||||
if (code != 0) {
|
||||
goto _err;
|
||||
}
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
|
||||
code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr),
|
||||
nData - sizeof(SSnapDataHdr));
|
||||
tDeleteSTqOffset(&info);
|
||||
if (code) goto _err;
|
||||
TSDB_CHECK_CODE(code, lino, end);
|
||||
tqInfo("vgId:%d, vnode tq offset write success", TD_VID(pTq->pVnode));
|
||||
|
||||
return code;
|
||||
|
||||
_err:
|
||||
tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
|
||||
end:
|
||||
if (code != 0){
|
||||
tqError("%s failed at %d, vnode tq offset write failed since %s", __func__, lino, tstrerror(code));
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1715,7 +1715,7 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo
|
|||
uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow);
|
||||
if (colType == 2) {
|
||||
SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type);
|
||||
tColDataGetValue(pColData, tRow.iRow, &colVal);
|
||||
TAOS_CHECK_GOTO(tColDataGetValue(pColData, tRow.iRow, &colVal), &lino, _exit);
|
||||
|
||||
SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
|
||||
if (!taosArrayPush(ctxArray, &updateCtx)) {
|
||||
|
|
|
|||
|
|
@ -680,7 +680,7 @@ static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSe
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) {
|
||||
static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) {
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
|
||||
|
|
@ -690,6 +690,8 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) {
|
|||
TFileSetArray *fsetArray = fs->fSetArrTmp;
|
||||
STFileSet *fset = NULL;
|
||||
const STFileOp *op;
|
||||
int32_t fid = INT32_MIN;
|
||||
TSKEY now = taosGetTimestampMs();
|
||||
TARRAY2_FOREACH_PTR(opArray, op) {
|
||||
if (!fset || fset->fid != op->fid) {
|
||||
STFileSet tfset = {.fid = op->fid};
|
||||
|
|
@ -708,6 +710,15 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) {
|
|||
|
||||
code = tsdbTFileSetEdit(fs->tsdb, fset, op);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
if (fid != op->fid) {
|
||||
fid = op->fid;
|
||||
if (etype == TSDB_FEDIT_COMMIT) {
|
||||
fset->lastCommit = now;
|
||||
} else if (etype == TSDB_FEDIT_COMPACT) {
|
||||
fset->lastCompact = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// remove empty empty stt level and empty file set
|
||||
|
|
@ -864,7 +875,7 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e
|
|||
fs->etype = etype;
|
||||
|
||||
// edit
|
||||
code = edit_fs(fs, opArray);
|
||||
code = edit_fs(fs, opArray, etype);
|
||||
TSDB_CHECK_CODE(code, lino, _exit);
|
||||
|
||||
// save fs
|
||||
|
|
@ -1288,6 +1299,12 @@ int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
extern bool tsdbShouldCompact(const STFileSet *pFileSet);
|
||||
|
||||
#ifndef TD_ENTERPRISE
|
||||
bool tsdbShouldCompact(const STFileSet *pFileSet) { return false; }
|
||||
#endif
|
||||
|
||||
static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) {
|
||||
STsdb *pTsdb = pReader->pTsdb;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
|
|
@ -1311,7 +1328,7 @@ static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) {
|
|||
// get file set details
|
||||
pReader->fid = pReader->pFileSet->fid;
|
||||
tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime);
|
||||
pReader->lastCompactTime = 0; // TODO
|
||||
pReader->lastCompactTime = pReader->pFileSet->lastCompact;
|
||||
pReader->totalSize = 0;
|
||||
for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) {
|
||||
STFileObj *fobj = pReader->pFileSet->farr[i];
|
||||
|
|
@ -1375,7 +1392,7 @@ int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *fie
|
|||
|
||||
fieldName = "should_compact";
|
||||
if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) {
|
||||
*(char *)value = 0; // TODO
|
||||
*(char *)value = tsdbShouldCompact(pReader->pFileSet);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -273,6 +273,15 @@ int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) {
|
|||
if (code) return code;
|
||||
}
|
||||
|
||||
// about compact and commit
|
||||
if (cJSON_AddNumberToObject(json, "last compact", fset->lastCompact) == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
if (cJSON_AddNumberToObject(json, "last commit", fset->lastCommit) == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -324,6 +333,20 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) {
|
|||
} else {
|
||||
return TSDB_CODE_FILE_CORRUPTED;
|
||||
}
|
||||
// about compact and commit
|
||||
item1 = cJSON_GetObjectItem(json, "last compact");
|
||||
if (cJSON_IsNumber(item1)) {
|
||||
(*fset)->lastCompact = item1->valuedouble;
|
||||
} else {
|
||||
(*fset)->lastCompact = 0;
|
||||
}
|
||||
|
||||
item1 = cJSON_GetObjectItem(json, "last commit");
|
||||
if (cJSON_IsNumber(item1)) {
|
||||
(*fset)->lastCommit = item1->valuedouble;
|
||||
} else {
|
||||
(*fset)->lastCommit = 0;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -467,6 +490,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f
|
|||
}
|
||||
}
|
||||
|
||||
fset2->lastCompact = fset1->lastCompact;
|
||||
fset2->lastCommit = fset1->lastCommit;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -522,6 +548,9 @@ int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **f
|
|||
if (code) return code;
|
||||
}
|
||||
|
||||
(*fset)->lastCompact = fset1->lastCompact;
|
||||
(*fset)->lastCommit = fset1->lastCommit;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -617,6 +646,9 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
|
|||
}
|
||||
}
|
||||
|
||||
(*fset)->lastCompact = fset1->lastCompact;
|
||||
(*fset)->lastCommit = fset1->lastCommit;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ struct STFileSet {
|
|||
int64_t maxVerValid;
|
||||
STFileObj *farr[TSDB_FTYPE_MAX]; // file array
|
||||
TSttLvlArray lvlArr[1]; // level array
|
||||
TSKEY lastCompact;
|
||||
TSKEY lastCommit;
|
||||
|
||||
bool mergeScheduled;
|
||||
SVATaskID mergeTask;
|
||||
|
|
|
|||
|
|
@ -123,7 +123,8 @@ static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int
|
|||
|
||||
pColData = &pBlock->aColData[slotId];
|
||||
|
||||
tColDataGetValue(pColData, irow, &cv);
|
||||
code = tColDataGetValue(pColData, irow, &cv);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
|
||||
pKey->numOfPKs = 1;
|
||||
pKey->pks[0].type = cv.value.type;
|
||||
|
|
@ -1603,7 +1604,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastPro
|
|||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
} else { // varchar/nchar type
|
||||
for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {
|
||||
tColDataGetValue(pData, j, &cv);
|
||||
code = tColDataGetValue(pData, j, &cv);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
}
|
||||
|
|
@ -5282,7 +5284,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
|
|||
|
||||
SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
|
||||
if (pData->cid == pSupInfo->colId[i]) {
|
||||
tColDataGetValue(pData, rowIndex, &cv);
|
||||
code = tColDataGetValue(pData, rowIndex, &cv);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
|
||||
TSDB_CHECK_CODE(code, lino, _end);
|
||||
j += 1;
|
||||
|
|
@ -5791,7 +5794,6 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
|
|||
|
||||
// make sure only release once
|
||||
void* p = pReader->pReadSnap;
|
||||
TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
|
||||
if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) {
|
||||
tsdbUntakeReadSnap2(pReader, p, false);
|
||||
pReader->pReadSnap = NULL;
|
||||
|
|
|
|||
|
|
@ -622,7 +622,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
|
|||
SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
|
||||
|
||||
if (pColData) {
|
||||
tColDataGetValue(pColData, pRow->iRow, pColVal);
|
||||
if (tColDataGetValue(pColData, pRow->iRow, pColVal) != 0){
|
||||
tsdbError("failed to tColDataGetValue");
|
||||
}
|
||||
} else {
|
||||
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
|
||||
}
|
||||
|
|
@ -645,7 +647,9 @@ void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
|
|||
SColData *pColData = &pBlock->aColData[i];
|
||||
if (pColData->cflag & COL_IS_KEY) {
|
||||
SColVal cv;
|
||||
tColDataGetValue(pColData, irow, &cv);
|
||||
if (tColDataGetValue(pColData, irow, &cv) != 0){
|
||||
break;
|
||||
}
|
||||
key->pks[key->numOfPKs] = cv.value;
|
||||
key->numOfPKs++;
|
||||
} else {
|
||||
|
|
@ -719,7 +723,9 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) {
|
|||
}
|
||||
|
||||
if (pIter->iColData <= pIter->pRow->pBlockData->nColData) {
|
||||
tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv);
|
||||
if (tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData - 1], pIter->pRow->iRow, &pIter->cv) != 0){
|
||||
return NULL;
|
||||
}
|
||||
++pIter->iColData;
|
||||
return &pIter->cv;
|
||||
} else {
|
||||
|
|
@ -1251,7 +1257,8 @@ static int32_t tBlockDataUpsertBlockRow(SBlockData *pBlockData, SBlockData *pBlo
|
|||
cv = COL_VAL_NONE(pColDataTo->cid, pColDataTo->type);
|
||||
if (flag == 0 && (code = tColDataAppendValue(pColDataTo, &cv))) goto _exit;
|
||||
} else {
|
||||
tColDataGetValue(pColDataFrom, iRow, &cv);
|
||||
code = tColDataGetValue(pColDataFrom, iRow, &cv);
|
||||
if (code) goto _exit;
|
||||
|
||||
if (flag) {
|
||||
code = tColDataUpdateValue(pColDataTo, &cv, flag > 0);
|
||||
|
|
|
|||
|
|
@ -191,7 +191,6 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->streamStateFillGetGroupKVByCur = streamStateFillGetGroupKVByCur;
|
||||
pStore->streamStateGetKVByCur = streamStateGetKVByCur;
|
||||
|
||||
pStore->streamStateSetFillInfo = streamStateSetFillInfo;
|
||||
pStore->streamStateClearExpiredState = streamStateClearExpiredState;
|
||||
|
||||
pStore->streamStateSessionAddIfNotExist = streamStateSessionAddIfNotExist;
|
||||
|
|
@ -243,7 +242,6 @@ void initStateStoreAPI(SStateStore* pStore) {
|
|||
pStore->streamStateBegin = streamStateBegin;
|
||||
pStore->streamStateCommit = streamStateCommit;
|
||||
pStore->streamStateDestroy = streamStateDestroy;
|
||||
pStore->streamStateDeleteCheckPoint = streamStateDeleteCheckPoint;
|
||||
pStore->streamStateReloadInfo = streamStateReloadInfo;
|
||||
pStore->streamStateCopyBackend = streamStateCopyBackend;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
|
|||
code = TSDB_CODE_VND_HASH_MISMATCH;
|
||||
goto _exit;
|
||||
} else if (mer1.me.type == TSDB_CHILD_TABLE) {
|
||||
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_LOCK);
|
||||
metaReaderDoInit(&mer2, pVnode->pMeta, META_READER_NOLOCK);
|
||||
if (metaReaderGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit;
|
||||
|
||||
tstrncpy(cfgRsp.stbName, mer2.me.name, TSDB_TABLE_NAME_LEN);
|
||||
|
|
@ -279,7 +279,8 @@ int32_t vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
|
|||
}
|
||||
} else {
|
||||
vError("vnodeGetTableCfg get invalid table type:%d", mer1.me.type);
|
||||
return TSDB_CODE_APP_ERROR;
|
||||
code = TSDB_CODE_APP_ERROR;
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
cfgRsp.numOfTags = schemaTag.nCols;
|
||||
|
|
|
|||
|
|
@ -607,9 +607,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
|
|||
}
|
||||
|
||||
vDebug("vgId:%d, start to process write request %s, index:%" PRId64 ", applied:%" PRId64 ", state.applyTerm:%" PRId64
|
||||
", conn.applyTerm:%" PRId64,
|
||||
", conn.applyTerm:%" PRId64 ", contLen:%d",
|
||||
TD_VID(pVnode), TMSG_INFO(pMsg->msgType), ver, pVnode->state.applied, pVnode->state.applyTerm,
|
||||
pMsg->info.conn.applyTerm);
|
||||
pMsg->info.conn.applyTerm, pMsg->contLen);
|
||||
|
||||
if (!(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm)) {
|
||||
return terrno = TSDB_CODE_INTERNAL_ERROR;
|
||||
|
|
@ -1415,7 +1415,8 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i
|
|||
SVAlterTbReq vAlterTbReq = {0};
|
||||
SVAlterTbRsp vAlterTbRsp = {0};
|
||||
SDecoder dc = {0};
|
||||
int32_t rcode = 0;
|
||||
int32_t code = 0;
|
||||
int32_t lino = 0;
|
||||
int32_t ret;
|
||||
SEncoder ec = {0};
|
||||
STableMetaRsp vMetaRsp = {0};
|
||||
|
|
@ -1431,7 +1432,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i
|
|||
if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) {
|
||||
vAlterTbRsp.code = TSDB_CODE_INVALID_MSG;
|
||||
tDecoderClear(&dc);
|
||||
rcode = -1;
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
|
|
@ -1439,7 +1439,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i
|
|||
if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) {
|
||||
vAlterTbRsp.code = terrno;
|
||||
tDecoderClear(&dc);
|
||||
rcode = -1;
|
||||
goto _exit;
|
||||
}
|
||||
tDecoderClear(&dc);
|
||||
|
|
@ -1449,6 +1448,31 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i
|
|||
vAlterTbRsp.pMeta = &vMetaRsp;
|
||||
}
|
||||
|
||||
if (vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL) {
|
||||
int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, vAlterTbReq.tbName);
|
||||
if (uid == 0) {
|
||||
vError("vgId:%d, %s failed at %s:%d since table %s not found", TD_VID(pVnode), __func__, __FILE__, __LINE__,
|
||||
vAlterTbReq.tbName);
|
||||
goto _exit;
|
||||
}
|
||||
|
||||
SArray* tbUids = taosArrayInit(4, sizeof(int64_t));
|
||||
void* p = taosArrayPush(tbUids, &uid);
|
||||
TSDB_CHECK_NULL(p, code, lino, _exit, terrno);
|
||||
|
||||
vDebug("vgId:%d, remove tags value altered table:%s from query table list", TD_VID(pVnode), vAlterTbReq.tbName);
|
||||
if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, false)) < 0) {
|
||||
vError("vgId:%d, failed to remove tbUid list since %s", TD_VID(pVnode), tstrerror(code));
|
||||
}
|
||||
|
||||
vDebug("vgId:%d, try to add table:%s in query table list", TD_VID(pVnode), vAlterTbReq.tbName);
|
||||
if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, true)) < 0) {
|
||||
vError("vgId:%d, failed to add tbUid list since %s", TD_VID(pVnode), tstrerror(code));
|
||||
}
|
||||
|
||||
taosArrayDestroy(tbUids);
|
||||
}
|
||||
|
||||
_exit:
|
||||
taosArrayDestroy(vAlterTbReq.pMultiTag);
|
||||
tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret);
|
||||
|
|
@ -1457,6 +1481,7 @@ _exit:
|
|||
if (tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp) != 0) {
|
||||
vError("vgId:%d, failed to encode alter table response", TD_VID(pVnode));
|
||||
}
|
||||
|
||||
tEncoderClear(&ec);
|
||||
if (vMetaRsp.pSchemas) {
|
||||
taosMemoryFree(vMetaRsp.pSchemas);
|
||||
|
|
@ -2145,7 +2170,7 @@ static int32_t vnodeConsolidateAlterHashRange(SVnode *pVnode, int64_t ver) {
|
|||
pVnode->config.hashBegin, pVnode->config.hashEnd, ver);
|
||||
|
||||
// TODO: trim meta of tables from TDB per hash range [pVnode->config.hashBegin, pVnode->config.hashEnd]
|
||||
code = metaTrimTables(pVnode->pMeta);
|
||||
code = metaTrimTables(pVnode->pMeta, ver);
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,29 +1,27 @@
|
|||
MESSAGE(STATUS "vnode unit test")
|
||||
MESSAGE(STATUS "tq unit test")
|
||||
|
||||
# GoogleTest requires at least C++11
|
||||
SET(CMAKE_CXX_STANDARD 11)
|
||||
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
|
||||
|
||||
# add_executable(tqTest "")
|
||||
# target_sources(tqTest
|
||||
# PRIVATE
|
||||
# "tqMetaTest.cpp"
|
||||
# )
|
||||
# target_include_directories(tqTest
|
||||
# PUBLIC
|
||||
# "${TD_SOURCE_DIR}/include/server/vnode/tq"
|
||||
# "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
|
||||
# )
|
||||
IF(NOT TD_WINDOWS)
|
||||
add_executable(tqTest tqTest.cpp)
|
||||
target_include_directories(tqTest
|
||||
PUBLIC
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
|
||||
)
|
||||
|
||||
# target_link_libraries(tqTest
|
||||
# tq
|
||||
# gtest_main
|
||||
# )
|
||||
# enable_testing()
|
||||
# add_test(
|
||||
# NAME tq_test
|
||||
# COMMAND tqTest
|
||||
# )
|
||||
TARGET_LINK_LIBRARIES(
|
||||
tqTest
|
||||
PUBLIC os util common vnode gtest_main
|
||||
)
|
||||
|
||||
enable_testing()
|
||||
|
||||
add_test(
|
||||
NAME tq_test
|
||||
COMMAND tqTest
|
||||
)
|
||||
ENDIF()
|
||||
|
||||
# ADD_EXECUTABLE(tsdbSmaTest tsdbSmaTest.cpp)
|
||||
# TARGET_LINK_LIBRARIES(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <vnodeInt.h>
|
||||
|
||||
#include <taoserror.h>
|
||||
#include <tglobal.h>
|
||||
#include <iostream>
|
||||
|
||||
#include <tmsg.h>
|
||||
#include <vnodeInt.h>
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wwrite-strings"
|
||||
#pragma GCC diagnostic ignored "-Wunused-function"
|
||||
#pragma GCC diagnostic ignored "-Wunused-variable"
|
||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||
|
||||
SDmNotifyHandle dmNotifyHdl = {.state = 0};
|
||||
|
||||
#include "tq.h"
|
||||
int main(int argc, char **argv) {
|
||||
testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
}
|
||||
|
||||
void tqWriteOffset() {
|
||||
TdFilePtr pFile = taosOpenFile(TQ_OFFSET_NAME, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
|
||||
|
||||
STqOffset offset = {.val = {.type = TMQ_OFFSET__LOG, .version = 8923}};
|
||||
strcpy(offset.subKey, "testtest");
|
||||
int32_t bodyLen;
|
||||
int32_t code;
|
||||
tEncodeSize(tEncodeSTqOffset, &offset, bodyLen, code);
|
||||
int32_t totLen = INT_BYTES + bodyLen;
|
||||
void* buf = taosMemoryCalloc(1, totLen);
|
||||
void* abuf = POINTER_SHIFT(buf, INT_BYTES);
|
||||
|
||||
*(int32_t*)buf = htonl(bodyLen);
|
||||
SEncoder encoder;
|
||||
tEncoderInit(&encoder, (uint8_t*)abuf, bodyLen);
|
||||
tEncodeSTqOffset(&encoder, &offset);
|
||||
taosWriteFile(pFile, buf, totLen);
|
||||
|
||||
taosMemoryFree(buf);
|
||||
|
||||
taosCloseFile(&pFile);
|
||||
}
|
||||
|
||||
TEST(testCase, tqOffsetTest) {
|
||||
STQ* pTq = (STQ*)taosMemoryCalloc(1, sizeof(STQ));
|
||||
pTq->path = taosStrdup("./");
|
||||
|
||||
pTq->pOffset = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_ENTRY_LOCK);
|
||||
taosHashSetFreeFp(pTq->pOffset, (FDelete)tDeleteSTqOffset);
|
||||
|
||||
tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0, 0, NULL);
|
||||
tdbTbOpen("tq.offset.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pOffsetStore, 0);
|
||||
|
||||
tqWriteOffset();
|
||||
tqOffsetRestoreFromFile(pTq, TQ_OFFSET_NAME);
|
||||
taosRemoveFile(TQ_OFFSET_NAME);
|
||||
tqClose(pTq);
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
|
@ -38,6 +38,8 @@ TDBlockBlobClient TDBlockBlobClient::CreateFromConnectionString(const std::strin
|
|||
return newClient;
|
||||
}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {}
|
||||
#if 0
|
||||
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, std::shared_ptr<StorageSharedKeyCredential> credential,
|
||||
const BlobClientOptions& options)
|
||||
: BlobClient(blobUrl, std::move(credential), options) {}
|
||||
|
|
@ -50,8 +52,6 @@ TDBlockBlobClient::TDBlockBlobClient(const std::string&
|
|||
TDBlockBlobClient::TDBlockBlobClient(const std::string& blobUrl, const BlobClientOptions& options)
|
||||
: BlobClient(blobUrl, options) {}
|
||||
|
||||
TDBlockBlobClient::TDBlockBlobClient(BlobClient blobClient) : BlobClient(std::move(blobClient)) {}
|
||||
|
||||
TDBlockBlobClient TDBlockBlobClient::WithSnapshot(const std::string& snapshot) const {
|
||||
TDBlockBlobClient newClient(*this);
|
||||
if (snapshot.empty()) {
|
||||
|
|
@ -74,47 +74,6 @@ TDBlockBlobClient TDBlockBlobClient::WithVersionId(const std::string& versionId)
|
|||
return newClient;
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobResult> TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content,
|
||||
const UploadBlockBlobOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions;
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
|
||||
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
|
||||
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
|
||||
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
|
||||
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
|
||||
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
|
||||
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
|
||||
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
|
||||
protocolLayerOptions.Tier = options.AccessTier;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
if (options.ImmutabilityPolicy.HasValue()) {
|
||||
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
|
||||
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
|
||||
}
|
||||
protocolLayerOptions.LegalHold = options.HasLegalHold;
|
||||
|
||||
return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
|
||||
const uint8_t* buffer, size_t bufferSize, const UploadBlockBlobFromOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
|
|
@ -270,6 +229,47 @@ Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom
|
|||
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
|
||||
std::move(commitBlockListResponse.RawResponse));
|
||||
}
|
||||
#endif
|
||||
Azure::Response<Models::UploadBlockBlobResult> TDBlockBlobClient::Upload(Azure::Core::IO::BodyStream& content,
|
||||
const UploadBlockBlobOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::UploadBlockBlobOptions protocolLayerOptions;
|
||||
if (options.TransactionalContentHash.HasValue()) {
|
||||
if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Md5) {
|
||||
protocolLayerOptions.TransactionalContentMD5 = options.TransactionalContentHash.Value().Value;
|
||||
} else if (options.TransactionalContentHash.Value().Algorithm == HashAlgorithm::Crc64) {
|
||||
protocolLayerOptions.TransactionalContentCrc64 = options.TransactionalContentHash.Value().Value;
|
||||
}
|
||||
}
|
||||
protocolLayerOptions.BlobContentType = options.HttpHeaders.ContentType;
|
||||
protocolLayerOptions.BlobContentEncoding = options.HttpHeaders.ContentEncoding;
|
||||
protocolLayerOptions.BlobContentLanguage = options.HttpHeaders.ContentLanguage;
|
||||
protocolLayerOptions.BlobContentMD5 = options.HttpHeaders.ContentHash.Value;
|
||||
protocolLayerOptions.BlobContentDisposition = options.HttpHeaders.ContentDisposition;
|
||||
protocolLayerOptions.BlobCacheControl = options.HttpHeaders.CacheControl;
|
||||
protocolLayerOptions.Metadata = std::map<std::string, std::string>(options.Metadata.begin(), options.Metadata.end());
|
||||
protocolLayerOptions.BlobTagsString = _detail::TagsToString(options.Tags);
|
||||
protocolLayerOptions.Tier = options.AccessTier;
|
||||
protocolLayerOptions.LeaseId = options.AccessConditions.LeaseId;
|
||||
protocolLayerOptions.IfModifiedSince = options.AccessConditions.IfModifiedSince;
|
||||
protocolLayerOptions.IfUnmodifiedSince = options.AccessConditions.IfUnmodifiedSince;
|
||||
protocolLayerOptions.IfMatch = options.AccessConditions.IfMatch;
|
||||
protocolLayerOptions.IfNoneMatch = options.AccessConditions.IfNoneMatch;
|
||||
protocolLayerOptions.IfTags = options.AccessConditions.TagConditions;
|
||||
if (m_customerProvidedKey.HasValue()) {
|
||||
protocolLayerOptions.EncryptionKey = m_customerProvidedKey.Value().Key;
|
||||
protocolLayerOptions.EncryptionKeySha256 = m_customerProvidedKey.Value().KeyHash;
|
||||
protocolLayerOptions.EncryptionAlgorithm = m_customerProvidedKey.Value().Algorithm.ToString();
|
||||
}
|
||||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
if (options.ImmutabilityPolicy.HasValue()) {
|
||||
protocolLayerOptions.ImmutabilityPolicyExpiry = options.ImmutabilityPolicy.Value().ExpiresOn;
|
||||
protocolLayerOptions.ImmutabilityPolicyMode = options.ImmutabilityPolicy.Value().PolicyMode;
|
||||
}
|
||||
protocolLayerOptions.LegalHold = options.HasLegalHold;
|
||||
|
||||
return _detail::BlockBlobClient::Upload(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom(
|
||||
const std::string& fileName, int64_t offset, int64_t size, const UploadBlockBlobFromOptions& options,
|
||||
|
|
@ -349,7 +349,7 @@ Azure::Response<Models::UploadBlockBlobFromResult> TDBlockBlobClient::UploadFrom
|
|||
return Azure::Response<Models::UploadBlockBlobFromResult>(std::move(result),
|
||||
std::move(commitBlockListResponse.RawResponse));
|
||||
}
|
||||
|
||||
#if 0
|
||||
Azure::Response<Models::UploadBlockBlobFromUriResult> TDBlockBlobClient::UploadFromUri(
|
||||
const std::string& sourceUri, const UploadBlockBlobFromUriOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
|
|
@ -396,7 +396,7 @@ Azure::Response<Models::UploadBlockBlobFromUriResult> TDBlockBlobClient::UploadF
|
|||
|
||||
return _detail::BlockBlobClient::UploadFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
#endif
|
||||
Azure::Response<Models::StageBlockResult> TDBlockBlobClient::StageBlock(const std::string& blockId,
|
||||
Azure::Core::IO::BodyStream& content,
|
||||
const StageBlockOptions& options,
|
||||
|
|
@ -419,7 +419,7 @@ Azure::Response<Models::StageBlockResult> TDBlockBlobClient::StageBlock(const st
|
|||
protocolLayerOptions.EncryptionScope = m_encryptionScope;
|
||||
return _detail::BlockBlobClient::StageBlock(*m_pipeline, m_blobUrl, content, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
#if 0
|
||||
Azure::Response<Models::StageBlockFromUriResult> TDBlockBlobClient::StageBlockFromUri(
|
||||
const std::string& blockId, const std::string& sourceUri, const StageBlockFromUriOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
|
|
@ -457,7 +457,7 @@ Azure::Response<Models::StageBlockFromUriResult> TDBlockBlobClient::StageBlockFr
|
|||
|
||||
return _detail::BlockBlobClient::StageBlockFromUri(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
#endif
|
||||
Azure::Response<Models::CommitBlockListResult> TDBlockBlobClient::CommitBlockList(
|
||||
const std::vector<std::string>& blockIds, const CommitBlockListOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
|
|
@ -492,7 +492,7 @@ Azure::Response<Models::CommitBlockListResult> TDBlockBlobClient::CommitBlockLis
|
|||
|
||||
return _detail::BlockBlobClient::CommitBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions, context);
|
||||
}
|
||||
|
||||
#if 0
|
||||
Azure::Response<Models::GetBlockListResult> TDBlockBlobClient::GetBlockList(const GetBlockListOptions& options,
|
||||
const Azure::Core::Context& context) const {
|
||||
_detail::BlockBlobClient::GetBlockBlobBlockListOptions protocolLayerOptions;
|
||||
|
|
@ -502,6 +502,7 @@ Azure::Response<Models::GetBlockListResult> TDBlockBlobClient::GetBlockList(cons
|
|||
return _detail::BlockBlobClient::GetBlockList(*m_pipeline, m_blobUrl, protocolLayerOptions,
|
||||
_internal::WithReplicaStatus(context));
|
||||
}
|
||||
#endif
|
||||
/*
|
||||
Azure::Response<Models::QueryBlobResult> TDBlockBlobClient::Query(const std::string& querySqlExpression,
|
||||
const QueryBlobOptions& options,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,210 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <queue>
|
||||
|
||||
// clang-format off
|
||||
#include "td_block_blob_client.hpp"
|
||||
#include "az.h"
|
||||
// clang-format on
|
||||
|
||||
using namespace Azure::Storage;
|
||||
using namespace Azure::Storage::Blobs;
|
||||
|
||||
extern int8_t tsS3Enabled;
|
||||
extern char tsS3BucketName[TSDB_FQDN_LEN];
|
||||
|
||||
static int32_t azInitEnv() {
|
||||
int32_t code = 0;
|
||||
|
||||
extern int8_t tsS3EpNum;
|
||||
|
||||
extern char tsS3Hostname[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
|
||||
extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
|
||||
|
||||
/* TCS parameter format
|
||||
tsS3Hostname[0] = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
tsS3AccessKeyId[0] = "<access-key-id/account-name>";
|
||||
tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
|
||||
tsS3BucketName = "<bucket/container-name>";
|
||||
*/
|
||||
|
||||
const char *hostname = "<endpoint>/<account-name>.blob.core.windows.net";
|
||||
const char *accessKeyId = "<access-key-id/account-name>";
|
||||
const char *accessKeySecret = "<access-key-secret/account-key>";
|
||||
const char *bucketName = "<bucket/container-name>";
|
||||
|
||||
if (hostname[0] != '<') {
|
||||
tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
|
||||
tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
|
||||
tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
|
||||
} else {
|
||||
const char *accountId = getenv("ablob_account_id");
|
||||
if (!accountId) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *accountSecret = getenv("ablob_account_secret");
|
||||
if (!accountSecret) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *containerName = getenv("ablob_container");
|
||||
if (!containerName) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
TAOS_STRCPY(&tsS3Hostname[0][0], accountId);
|
||||
TAOS_STRCAT(&tsS3Hostname[0][0], ".blob.core.windows.net");
|
||||
TAOS_STRCPY(&tsS3AccessKeyId[0][0], accountId);
|
||||
TAOS_STRCPY(&tsS3AccessKeySecret[0][0], accountSecret);
|
||||
TAOS_STRCPY(tsS3BucketName, containerName);
|
||||
}
|
||||
|
||||
tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
|
||||
|
||||
tsS3Enabled = true;
|
||||
|
||||
return code;
|
||||
}
|
||||
|
||||
// TEST(AzTest, DISABLED_InterfaceTest) {
|
||||
TEST(AzETest, InterfaceTest) {
|
||||
int code = 0;
|
||||
bool check = false;
|
||||
bool withcp = false;
|
||||
|
||||
code = azInitEnv();
|
||||
if (code) {
|
||||
std::cout << "ablob env init failed with: " << code << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
GTEST_ASSERT_EQ(tsS3Enabled, 1);
|
||||
|
||||
code = azBegin();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azCheckCfg();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
const int size = 4096;
|
||||
char data[size] = {0};
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
const char object_name[] = "azut.bin";
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azPutObjectFromFileOffset(path, object_name, 0, size);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
code = azGetObjectBlock(object_name, 0, size, check, &pBlock);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
|
||||
}
|
||||
|
||||
taosMemoryFree(pBlock);
|
||||
|
||||
code = azGetObjectToFile(object_name, path_download);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
azDeleteObjectsByPrefix(object_name);
|
||||
// list object to check
|
||||
|
||||
code = azPutObjectFromFile2(path, object_name, withcp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azGetObjectsByPrefix(object_name, tsTempDir);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
TDBlockBlobClient blobClient =
|
||||
TDBlockBlobClient::CreateFromConnectionString(std::getenv("ablob_cs"), std::string(tsS3BucketName), object_name);
|
||||
|
||||
const char *object_name_arr[] = {object_name};
|
||||
code = azDeleteObjects(object_name_arr, 1);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
azEnd();
|
||||
}
|
||||
|
|
@ -199,3 +199,132 @@ TEST(AzTest, InterfaceTest) {
|
|||
|
||||
azEnd();
|
||||
}
|
||||
|
||||
// TEST(AzTest, DISABLED_InterfaceTestBig) {
|
||||
TEST(AzTest, InterfaceTestBig) {
|
||||
int code = 0;
|
||||
bool check = false;
|
||||
bool withcp = false;
|
||||
|
||||
code = azInitEnv();
|
||||
if (code) {
|
||||
std::cout << "ablob env init failed with: " << code << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
GTEST_ASSERT_EQ(tsS3Enabled, 1);
|
||||
|
||||
code = azBegin();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azCheckCfg();
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
const int size = 256 * 1024 * 1024 + 1;
|
||||
char *data = (char *)taosMemoryCalloc(1, size);
|
||||
if (!data) {
|
||||
std::cout << "code: " << code << "terrno: " << terrno << std::endl;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
data[i * 2 + 1] = 1;
|
||||
}
|
||||
|
||||
const char object_name[] = "azut.bin";
|
||||
char path[PATH_MAX] = {0};
|
||||
char path_download[PATH_MAX] = {0};
|
||||
int ds_len = strlen(TD_DIRSEP);
|
||||
int tmp_len = strlen(tsTempDir);
|
||||
|
||||
(void)snprintf(path, PATH_MAX, "%s", tsTempDir);
|
||||
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP);
|
||||
(void)snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", object_name);
|
||||
} else {
|
||||
(void)snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", object_name);
|
||||
}
|
||||
|
||||
tstrncpy(path_download, path, strlen(path) + 1);
|
||||
tstrncpy(path_download + strlen(path), ".download", strlen(".download") + 1);
|
||||
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_WRITE_THROUGH);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
int n = taosWriteFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azPutObjectFromFileOffset(path, object_name, 0, size);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
uint8_t *pBlock = NULL;
|
||||
code = azGetObjectBlock(object_name, 0, size, check, &pBlock);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(pBlock[i * 2 + 1], 1);
|
||||
}
|
||||
|
||||
taosMemoryFree(pBlock);
|
||||
|
||||
code = azGetObjectToFile(object_name, path_download);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
azDeleteObjectsByPrefix(object_name);
|
||||
// list object to check
|
||||
|
||||
code = azPutObjectFromFile2(path, object_name, withcp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
code = azGetObjectsByPrefix(object_name, tsTempDir);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
{
|
||||
TdFilePtr fp = taosOpenFile(path, TD_FILE_READ);
|
||||
GTEST_ASSERT_NE(fp, nullptr);
|
||||
|
||||
(void)memset(data, 0, size);
|
||||
|
||||
int64_t n = taosReadFile(fp, data, size);
|
||||
GTEST_ASSERT_EQ(n, size);
|
||||
|
||||
code = taosCloseFile(&fp);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
for (int i = 0; i < size / 2; ++i) {
|
||||
GTEST_ASSERT_EQ(data[i * 2], 0);
|
||||
GTEST_ASSERT_EQ(data[i * 2 + 1], 1);
|
||||
}
|
||||
}
|
||||
|
||||
const char *object_name_arr[] = {object_name};
|
||||
code = azDeleteObjects(object_name_arr, 1);
|
||||
GTEST_ASSERT_EQ(code, 0);
|
||||
|
||||
taosMemoryFree(data);
|
||||
|
||||
azEnd();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,6 @@ target_link_libraries(
|
|||
PRIVATE os util transport qcom nodes
|
||||
)
|
||||
|
||||
#if(${BUILD_TEST})
|
||||
# ADD_SUBDIRECTORY(test)
|
||||
#endif(${BUILD_TEST})
|
||||
if(${BUILD_TEST} AND NOT ${TD_WINDOWS})
|
||||
ADD_SUBDIRECTORY(test)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -424,6 +424,7 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName,
|
|||
CTG_RET(TSDB_CODE_SUCCESS);
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) {
|
||||
SVgroupInfo vgroupInfo = {0};
|
||||
STableCfg* pCfg = NULL;
|
||||
|
|
@ -474,6 +475,7 @@ _return:
|
|||
|
||||
CTG_RET(code);
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) {
|
||||
STableMeta* tbMeta = NULL;
|
||||
|
|
@ -1695,6 +1697,7 @@ _return:
|
|||
CTG_API_LEAVE(code);
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
|
|
@ -1709,6 +1712,7 @@ _return:
|
|||
|
||||
CTG_API_LEAVE(code);
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) {
|
||||
CTG_API_ENTER();
|
||||
|
|
@ -1845,7 +1849,7 @@ _return:
|
|||
CTG_API_LEAVE(code);
|
||||
}
|
||||
|
||||
|
||||
#if 0
|
||||
int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) {
|
||||
CTG_API_ENTER();
|
||||
|
||||
|
|
@ -1860,6 +1864,7 @@ _return:
|
|||
|
||||
CTG_API_LEAVE(code);
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pViewName, STableMeta** pTableMeta) {
|
||||
CTG_API_ENTER();
|
||||
|
|
@ -1981,10 +1986,19 @@ void catalogDestroy(void) {
|
|||
}
|
||||
|
||||
if (gCtgMgmt.cacheTimer) {
|
||||
if (taosTmrStop(gCtgMgmt.cacheTimer)) {
|
||||
qTrace("stop catalog cache timer may failed");
|
||||
if (!taosTmrStop(gCtgMgmt.cacheTimer)) {
|
||||
/*
|
||||
qDebug("catalog cacheTimer %" PRIuPTR " not stopped", (uintptr_t)gCtgMgmt.cacheTimer);
|
||||
|
||||
while (!taosTmrIsStopped(&gCtgMgmt.cacheTimer)) {
|
||||
taosMsleep(1);
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
qDebug("catalog cacheTimer %" PRIuPTR " is stopped", (uintptr_t)gCtgMgmt.cacheTimer);
|
||||
gCtgMgmt.cacheTimer = NULL;
|
||||
|
||||
taosTmrCleanUp(gCtgMgmt.timer);
|
||||
gCtgMgmt.timer = NULL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -480,6 +480,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
|
|||
|
||||
dbCache = (SCtgDBCache *)pIter;
|
||||
|
||||
CTG_LOCK(CTG_READ, &dbCache->dbLock);
|
||||
|
||||
dbFName = taosHashGetKey(pIter, &len);
|
||||
|
||||
int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0;
|
||||
|
|
@ -509,6 +511,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
|
|||
hashMethod, hashPrefix, hashSuffix, vgNum);
|
||||
|
||||
if (dbCache->vgCache.vgInfo) {
|
||||
CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock);
|
||||
|
||||
int32_t i = 0;
|
||||
void *pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, NULL);
|
||||
while (pVgIter) {
|
||||
|
|
@ -524,6 +528,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
|
|||
|
||||
pVgIter = taosHashIterate(dbCache->vgCache.vgInfo->vgHash, pVgIter);
|
||||
}
|
||||
|
||||
CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock);
|
||||
}
|
||||
|
||||
if (dbCache->cfgCache.cfgInfo) {
|
||||
|
|
@ -544,6 +550,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
|
|||
pCfg->schemaless, pCfg->sstTrigger);
|
||||
}
|
||||
|
||||
CTG_UNLOCK(CTG_READ, &dbCache->dbLock);
|
||||
|
||||
++i;
|
||||
pIter = taosHashIterate(dbHash, pIter);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,12 +149,13 @@ void ctgTestInitLogFile() {
|
|||
return;
|
||||
}
|
||||
|
||||
const char *defaultLogFileNamePrefix = "taoslog";
|
||||
const char *defaultLogFileNamePrefix = "catalogTest";
|
||||
const int32_t maxLogFileNum = 10;
|
||||
|
||||
tsAsyncLog = 0;
|
||||
qDebugFlag = 159;
|
||||
tmrDebugFlag = 159;
|
||||
tsNumOfLogLines = 1000000000;
|
||||
TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH);
|
||||
|
||||
(void)ctgdEnableDebug("api", true);
|
||||
|
|
@ -1839,7 +1840,7 @@ TEST(tableMeta, updateStbMeta) {
|
|||
while (true) {
|
||||
uint64_t n = 0;
|
||||
ASSERT(0 == ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n));
|
||||
if (n != 3) {
|
||||
if (n < 3) {
|
||||
taosMsleep(50);
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -2992,6 +2993,7 @@ TEST(apiTest, catalogGetTableIndex_test) {
|
|||
catalogDestroy();
|
||||
}
|
||||
|
||||
|
||||
TEST(apiTest, catalogGetDBCfg_test) {
|
||||
struct SCatalog *pCtg = NULL;
|
||||
SRequestConnInfo connInfo = {0};
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue