xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into fix/TD-31016.1

This commit is contained in:
dapan1121 2024-07-18 19:15:46 +08:00
parent ecc2ae7757 44406c3fd3
commit 989c31f0a7
136 changed files with 6173 additions and 3909 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -401,7 +401,7 @@ pipeline {
} }
} }
stage('linux test') { stage('linux test') {
agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || slave1_59 || slave1_63 || worker03 || slave215 || slave217 || slave219 "} agent{label "slave1_47 || slave1_48 || slave1_49 || slave1_50 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() } options { skipDefaultCheckout() }
when { when {
changeRequest() changeRequest()

1
docs/en/12-taos-sql/03-table.md
View File

@ -49,6 +49,7 @@ table_option: {
7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive. 7. Escape character "\`" can be used to avoid the conflict between table names and reserved keywords, above rules will be bypassed when using escape character on table names, but the upper limit for the name length is still valid. The table names specified using escape character are case sensitive.
For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally. For example \`aBc\` and \`abc\` are different table names but `abc` and `aBc` are same table names because they are both converted to `abc` internally.
Only ASCII visible characters can be used with escape character. Only ASCII visible characters can be used with escape character.
8. For the details of using `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress).
**Parameter description** **Parameter description**

20
docs/en/12-taos-sql/04-stable.md
View File

@ -13,17 +13,29 @@ create_definition:
col_name column_definition col_name column_definition
column_definition: column_definition:
type_name type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
table_option: {
COMMENT 'string_value'
| SMA(col_name [, col_name] ...)
| TTL value
}
``` ```
**More explanations** **More explanations**
- Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column. 1. Each supertable can have a maximum of 4096 columns, including tags. The minimum number of columns is 3: a timestamp column used as the key, one tag column, and one data column.
- The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns: 2. Since version 3.3.0.0, besides the timestamp, you can specify another column as primary key using `PRIMARY KEY` keyword, the column specified using `primary key` must be type of integer or varchar.
2. The TAGS keyword defines the tag columns for the supertable. The following restrictions apply to tag columns:
- A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column. - A tag column can use the TIMESTAMP data type, but the values in the column must be fixed numbers. Timestamps including formulae, such as "now + 10s", cannot be stored in a tag column.
- The name of a tag column cannot be the same as the name of any other column. - The name of a tag column cannot be the same as the name of any other column.
- The name of a tag column cannot be a reserved keyword. - The name of a tag column cannot be a reserved keyword.
- Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB. - Each supertable must contain between 1 and 128 tags. The total length of the TAGS keyword cannot exceed 16 KB.
- For more information about table parameters, see Create a Table. 3. Regarding how to use `ENCODE` and `COMPRESS`, please refer to [Encode and Compress for Column](../compress).
3. For more information about table parameters, see [Create a Table](../table).
## View a Supertable ## View a Supertable

2
docs/en/12-taos-sql/12-distinguished.md
View File

@ -38,7 +38,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## Windowed Queries ## Windowed Queries
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are four kinds of windows: time window, status window, session window, and event window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows: Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are five kinds of windows: time window, status window, session window, event window, and count window. There are two kinds of time windows: sliding window and flip time/tumbling window. The syntax of window clause is as follows:
```sql ```sql
window_clause: { window_clause: {

0
docs/en/21-tdinternal/08-compress.md → docs/en/12-taos-sql/31-compress.md
View File

51
docs/en/14-reference/12-config/index.md
View File

@ -729,6 +729,57 @@ The charset that takes effect is UTF-8.
| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | | Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed |
| Default | -1 | | Default | -1 |
### fPrecision
| Attribute | Description |
| -------- | -------------------------------- |
| Application | Server Only |
| Meaning | Compression precision for float data type |
| Value Range | 0.1 ~ 0.00000001 |
| Default | 0.00000001 |
| Note | The floating value below this setting will be cut off |
### dPrecision
| Attribute | Description |
| -------- | -------------------------------- |
| Applicable | Server Only |
| Meaning | Compression precision for double data type |
| Value Range | 0.1 ~ 0.0000000000000001 |
| Default | 0.0000000000000001 |
| Note | The floating value below this setting will be cut off |
### lossyColumn
| Attribute | Description |
| -------- | -------------------------------- |
| Applicable | Server Only |
| Meaning | Enable TSZ lossy compression for float and/or double |
| Value Range | float, double |
| Default | none: disable TSZ lossy compression |
**补充说明**
1. It's only available since 3.2.0.0 version, and can't downgrade to previous version once upgrading to 3.2.0.0 and enabling this parameter
2. TSZ compression algorithm compresses data based on data prediction technique, so it's more suitable for data with specific pattern
3. TSZ compression algorithm may take longer time but it has better compression ratio, so it's suitable when you have enough CPU resources and are more sensitive to disk occupation
4. Example: enable TSZ for both float and double
```shell
lossyColumns float|double
```
5. After configuring, taosd service needs to restarted. After restarting, if you see the following output in taosd logfile, it means the function has been enabled
```sql
02/22 10:49:27.607990 00002933 UTL lossyColumns float|double
```
### ifAdtFse
| Attribute | Description |
| -------- | -------------------------------- |
| Applicable | Server Only |
| Meaning | Replace HUFFMAN with FSE in TSZ, FSE is faster when compressing but slower when uncompressing |
| Value Range | 0: Use HUFFMAN, 1: Use FSE |
| Default | 0: Use HUFFMAN |
## Other Parameters ## Other Parameters

1
docs/zh/12-taos-sql/03-table.md
View File

@ -49,6 +49,7 @@ table_option: {
6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。 6. 使用数据类型 BINARY/NCHAR/GEOMETRY，需指定其最长的字节数，如 BINARY(20)，表示 20 字节。
7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一， 7. 为了兼容支持更多形式的表名，TDengine 引入新的转义符 "\`"，可以让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。但是同样具有长度限制要求。使用转义字符以后，不再对转义字符中的内容进行大小写统一，
例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。 例如：\`aBc\` 和 \`abc\` 是不同的表名，但是 abc 和 aBc 是相同的表名。
8. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考[按列压缩](../compress)
**参数说明** **参数说明**

19
docs/zh/12-taos-sql/04-stable.md
View File

@ -13,17 +13,28 @@ create_definition:
col_name column_definition col_name column_definition
column_definition: column_definition:
type_name type_name [comment 'string_value'] [PRIMARY KEY] [ENCODE 'encode_type'] [COMPRESS 'compress_type'] [LEVEL 'level_type']
table_options:
table_option ...
table_option: {
COMMENT 'string_value'
| SMA(col_name [, col_name] ...)
| TTL value
}
``` ```
**使用说明** **使用说明**
- 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。 1. 超级表中列的最大个数为 4096，需要注意，这里的 4096 是包含 TAG 列在内的，最小个数为 3，包含一个时间戳主键、一个 TAG 列和一个数据列。
- TAGS语法指定超级表的标签列，标签列需要遵循以下约定： 2. 除时间戳主键列之外，还可以通过 PRIMARY KEY 关键字指定第二列为额外的主键列。被指定为主键列的第二列必须为整型或字符串类型（varchar）
3. TAGS语法指定超级表的标签列，标签列需要遵循以下约定：
- TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。 - TAGS 中的 TIMESTAMP 列写入数据时需要提供给定值，而暂不支持四则运算，例如 NOW + 10s 这类表达式。
- TAGS 列名不能与其他列名相同。 - TAGS 列名不能与其他列名相同。
- TAGS 列名不能为预留关键字。 - TAGS 列名不能为预留关键字。
- TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。 - TAGS 最多允许 128 个，至少 1 个，总长度不超过 16 KB。
- 关于表参数的详细说明，参见 CREATE TABLE 中的介绍。 4. 关于 `ENCODE` 和 `COMPRESS` 的使用，请参考 [按列压缩](../compress)
5. 关于 table_option 中的参数说明，请参考 [建表 SQL 说明](../table)
## 查看超级表 ## 查看超级表

2
docs/zh/12-taos-sql/12-distinguished.md
View File

@ -39,7 +39,7 @@ select _wstart, tbname, avg(voltage) from meters partition by tbname interval(10
## 窗口切分查询 ## 窗口切分查询
TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）四种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 TDengine 支持按时间窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）、事件窗口（event window）、计数窗口（count window）五种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。
窗口子句语法如下： 窗口子句语法如下：

1
docs/zh/21-tdinternal/08-compress.md → docs/zh/12-taos-sql/31-compress.md
View File

@ -1,5 +1,6 @@
--- ---
title: 可配置压缩算法 title: 可配置压缩算法
sidebar_label: 可配置压缩
description: 可配置压缩算法 description: 可配置压缩算法
--- ---

54
docs/zh/14-reference/12-config/index.md
View File

@ -784,6 +784,57 @@ charset 的有效值是 UTF-8。
| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | | 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 |
| 缺省值 | -1 | | 缺省值 | -1 |
### fPrecision
FLOAT 类型压缩精度控制：
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 设置 float 类型浮点数压缩精度 |
| 取值范围 | 0.1 ~ 0.00000001 |
| 缺省值 | 0.00000001 |
| 补充说明 | 小于此值的浮点数尾数部分将被截取 |
### dPrecision
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 设置 double 类型浮点数压缩精度 |
| 取值范围 | 0.1 ~ 0.0000000000000001 |
| 缺省值 | 0.0000000000000001 |
| 补充说明 | 小于此值的浮点数尾数部分将被截取 |
### lossyColumn
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 对 float 和/或 double 类型启用 TSZ 有损压缩 |
| 取值范围 | float, double |
| 缺省值 | none：表示关闭无损压缩 |
**补充说明**
1. 在 3.2.0.0 ~ 3.3.0.0（不包含）版本生效，启用该参数后不能回退到升级前的版本
2. TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据
3. TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用
4. 示例：对 float 和 double 类型都启用有损压缩
```shell
lossyColumns float|double
```
5. 配置需重启服务生效，重启如果在 taosd 日志中看到以下内容，表明配置已生效：
```sql
02/22 10:49:27.607990 00002933 UTL lossyColumns float|double
```
### ifAdtFse
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 在启用 TSZ 有损压缩时，使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 |
| 取值范围 | 0：关闭 1：打开 |
| 缺省值 | 0：关闭 |
## 3.0 中有效的配置参数列表 ## 3.0 中有效的配置参数列表
| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 |
@ -922,8 +973,5 @@ charset 的有效值是 UTF-8。
| 73 | probeSeconds | 是 | 否 | 3.0 行为未知 | | 73 | probeSeconds | 是 | 否 | 3.0 行为未知 |
| 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | | 74 | probeKillSeconds | 是 | 否 | 3.0 行为未知 |
| 75 | probeInterval | 是 | 否 | 3.0 行为未知 | | 75 | probeInterval | 是 | 否 | 3.0 行为未知 |
| 76 | lossyColumns | 是 | 否 | 3.0 行为未知 |
| 77 | fPrecision | 是 | 否 | 3.0 行为未知 |
| 78 | dPrecision | 是 | 否 | 3.0 行为未知 |
| 79 | maxRange | 是 | 否 | 3.0 行为未知 | | 79 | maxRange | 是 | 否 | 3.0 行为未知 |
| 80 | range | 是 | 否 | 3.0 行为未知 | | 80 | range | 是 | 否 | 3.0 行为未知 |

69
docs/zh/21-tdinternal/07-tsz.md
View File

@ -1,69 +0,0 @@
---
title: TSZ 压缩算法
description: TDengine 对浮点数进行高效压缩的算法
---
TSZ 压缩算法是 TDengine 为浮点数据类型提供的可选压缩算法，可以实现浮点数有损至无损全状态压缩，相比默认压缩算法， TSZ 压缩算法压缩率更高，即使切至无损状态，压缩率也会比默认压缩高一倍。
## 适合场景
- TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据
- TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用
## 使用步骤
- TDengine 支持版本为 3.2.0.0 或以上
- 开启选项
在 taos.cfg 配置中增加以下内容，即可开启 TSZ 压缩算法，功能打开后，会替换默认算法。
以下表示字段类型是 float 及 double 类型都使用此压缩算法，也可以单独只配置一个
```sql
lossyColumns float|double
```
- 配置需重启服务生效
- Taosd 日志输出以下内容，表明功能已生效：
```sql
02/22 10:49:27.607990 00002933 UTL lossyColumns float|double
```
## 配置参数
### fPrecision
FLOAT 类型精度控制：
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 设置 float 类型浮点数压缩精度 |
| 取值范围 | 0.1 ~ 0.00000001 |
| 缺省值 | 0.00000001 |
| 补充说明 | 小于此值的浮点数尾数部分将被截取 |
### dPrecision
DOUBLE 类型精度控制：
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 设置 double 类型浮点数压缩精度 |
| 取值范围 | 0.1 ~ 0.0000000000000001 |
| 缺省值 | 0.0000000000000001 |
| 补充说明 | 小于此值的浮点数尾数部分将被截取 |
### ifAdtFse
TSZ 压缩中可选择的算法 FSE，默认为 HUFFMAN：
| 属性 | 说明 |
| -------- | -------------------------------- |
| 适用范围 | 服务器端 |
| 含义 | 使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 |
| 取值范围 | 0：关闭 1：打开 |
| 缺省值 | 0：关闭 |
## 注意事项
- 打开 TSZ 后生成的存储数据格式，回退至 3.2.0.0 之前的版本，数据将不能被识别

4
include/common/cos_cp.h
View File

@ -60,8 +60,8 @@ typedef struct {
} SCheckpoint; } SCheckpoint;
int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint); int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
void cos_cp_close(TdFilePtr fd); int32_t cos_cp_close(TdFilePtr fd);
void cos_cp_remove(char const* filepath); int32_t cos_cp_remove(char const* filepath);
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint); int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
int32_t cos_cp_dump(SCheckpoint* checkpoint); int32_t cos_cp_dump(SCheckpoint* checkpoint);

2
include/common/tdataformat.h
View File

@ -136,7 +136,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); void tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// SRowIter ================================ // SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);

2
include/common/tmisce.h
View File

@ -31,7 +31,7 @@ typedef struct SCorEpSet {
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len); int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t len);
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp);
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2);
void epsetAssign(SEpSet* dst, const SEpSet* pSrc); void epsetAssign(SEpSet* dst, const SEpSet* pSrc);

4
include/common/tmsgcb.h
View File

@ -17,13 +17,13 @@
#define _TD_COMMON_MSG_CB_H_ #define _TD_COMMON_MSG_CB_H_
#include "os.h" #include "os.h"
#include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SRpcMsg SRpcMsg; typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper; typedef struct SMgmtWrapper SMgmtWrapper;
typedef struct SRpcHandleInfo SRpcHandleInfo; typedef struct SRpcHandleInfo SRpcHandleInfo;
@ -46,7 +46,7 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
typedef void (*SendRspFp)(SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg);
typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*RegisterBrokenLinkArgFp)(struct SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc); typedef void (*ReportStartup)(const char* name, const char* desc);

2
include/common/tname.h
View File

@ -81,7 +81,7 @@ typedef struct {
// uint64_t uid; // child table uid, may be useful // uint64_t uid; // child table uid, may be useful
} RandTableName; } RandTableName;
void buildChildTableName(RandTableName* rName); int32_t buildChildTableName(RandTableName* rName);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
include/libs/audit/audit.h
View File

@ -50,7 +50,7 @@ typedef struct {
int32_t auditInit(const SAuditCfg *pCfg); int32_t auditInit(const SAuditCfg *pCfg);
void auditCleanup(); void auditCleanup();
void auditSend(SJson *pJson); int32_t auditSend(SJson *pJson);
void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,
char *detail, int32_t len); char *detail, int32_t len);
void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2,

14
include/libs/executor/storageapi.h
View File

@ -105,7 +105,7 @@ typedef struct SMTbCursor {
} SMTbCursor; } SMTbCursor;
typedef struct SMCtbCursor { typedef struct SMCtbCursor {
SMeta* pMeta; struct SMeta* pMeta;
void* pCur; void* pCur;
tb_uid_t suid; tb_uid_t suid;
void* pKey; void* pKey;
@ -134,7 +134,7 @@ typedef struct SMetaTableInfo {
} SMetaTableInfo; } SMetaTableInfo;
typedef struct SSnapContext { typedef struct SSnapContext {
SMeta* pMeta; struct SMeta* pMeta;
int64_t snapVersion; int64_t snapVersion;
void* pCur; void* pCur;
int64_t suid; int64_t suid;
@ -178,7 +178,7 @@ typedef struct TsdReader {
int32_t (*tsdNextDataBlock)(); int32_t (*tsdNextDataBlock)();
int32_t (*tsdReaderRetrieveBlockSMAInfo)(); int32_t (*tsdReaderRetrieveBlockSMAInfo)();
SSDataBlock *(*tsdReaderRetrieveDataBlock)(); int32_t (*tsdReaderRetrieveDataBlock)();
void (*tsdReaderReleaseDataBlock)(); void (*tsdReaderReleaseDataBlock)();
@ -381,7 +381,7 @@ typedef struct SStateStore {
void** ppVal, int32_t* pVLen); void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
@ -389,12 +389,12 @@ typedef struct SStateStore {
void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*updateInfoDestroy)(SUpdateInfo* pInfo);
void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count); void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count); int32_t (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo); void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo); void (*updateInfoDestoryColseWinSBF)(SUpdateInfo* pInfo);
int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo); int32_t (*updateInfoSerialize)(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo); int32_t (*updateInfoDeserialize)(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key); SStreamStateCur* (*streamStateSessionSeekKeyNext)(SStreamState* pState, const SSessionKey* key);

8
include/libs/parser/parser.h
View File

@ -149,10 +149,10 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char*
void qDestroyBoundColInfo(void* pInfo); void qDestroyBoundColInfo(void* pInfo);
SQuery* smlInitHandle(); int32_t smlInitHandle(SQuery** query);
int32_t smlBuildRow(STableDataCxt* pTableCxt); int32_t smlBuildRow(STableDataCxt* pTableCxt);
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index);
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt);
void clearColValArraySml(SArray* pCols); void clearColValArraySml(SArray* pCols);
int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
@ -163,8 +163,8 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCr
int numFields, bool needChangeLength, char* errstr, int32_t errstrLen); int numFields, bool needChangeLength, char* errstr, int32_t errstrLen);
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); int32_t serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap, SArray** pOut);
void destoryCatalogReq(SCatalogReq *pCatalogReq); void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr); bool isPrimaryKeyImpl(SNode* pExpr);
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo); int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo);

2
include/libs/stream/tstream.h
View File

@ -237,6 +237,8 @@ typedef struct {
int64_t stbUid; int64_t stbUid;
char stbFullName[TSDB_TABLE_FNAME_LEN]; char stbFullName[TSDB_TABLE_FNAME_LEN];
SSchemaWrapper* pSchemaWrapper; SSchemaWrapper* pSchemaWrapper;
SSchemaWrapper* pTagSchema;
bool autoCreateCtb;
void* vnode; // not available to encoder and decoder void* vnode; // not available to encoder and decoder
FTbSink* tbSinkFunc; FTbSink* tbSinkFunc;
STSchema* pTSchema; STSchema* pTSchema;

28
include/libs/stream/tstreamUpdate.h
View File

@ -15,29 +15,29 @@
#ifndef _TSTREAMUPDATE_H_ #ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_ #define _TSTREAMUPDATE_H_
#include "storageapi.h"
#include "taosdef.h" #include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tcommon.h" #include "tcommon.h"
#include "tmsg.h" #include "tmsg.h"
#include "storageapi.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
SUpdateInfo *updateInfoInitP(SInterval *pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t updateInfoInitP(SInterval* pInterval, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen); int32_t updateInfoInit(int64_t interval, int32_t precision, int64_t watermark, bool igUp, int8_t pkType, int32_t pkLen, SUpdateInfo** ppInfo);
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol, int32_t primaryKeyCol); TSKEY updateInfoFillBlockData(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol, int32_t primaryKeyCol);
bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool updateInfoIsUpdated(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid); bool updateInfoIsTableInserted(SUpdateInfo* pInfo, int64_t tbUid);
void updateInfoDestroy(SUpdateInfo *pInfo); void updateInfoDestroy(SUpdateInfo* pInfo);
void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); void updateInfoAddCloseWindowSBF(SUpdateInfo* pInfo);
void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); void updateInfoDestoryColseWinSBF(SUpdateInfo* pInfo);
int32_t updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *pInfo); int32_t updateInfoSerialize(void* buf, int32_t bufLen, const SUpdateInfo* pInfo, int32_t* pLen);
int32_t updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo); int32_t updateInfoDeserialize(void* buf, int32_t bufLen, SUpdateInfo* pInfo);
void windowSBfDelete(SUpdateInfo *pInfo, uint64_t count); void windowSBfDelete(SUpdateInfo* pInfo, uint64_t count);
void windowSBfAdd(SUpdateInfo *pInfo, uint64_t count); int32_t windowSBfAdd(SUpdateInfo* pInfo, uint64_t count);
bool isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len); bool isIncrementalTimeStamp(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts, void* pPkVal, int32_t len);
#ifdef __cplusplus #ifdef __cplusplus
} }

3
include/libs/sync/sync.h
View File

@ -70,7 +70,6 @@ typedef int64_t SyncIndex;
typedef int64_t SyncTerm; typedef int64_t SyncTerm;
typedef struct SSyncNode SSyncNode; typedef struct SSyncNode SSyncNode;
typedef struct SWal SWal;
typedef struct SSyncRaftEntry SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry;
typedef enum { typedef enum {
@ -238,7 +237,7 @@ typedef struct SSyncInfo {
int32_t batchSize; int32_t batchSize;
SSyncCfg syncCfg; SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN]; char path[TSDB_FILENAME_LEN];
SWal* pWal; struct SWal* pWal;
SSyncFSM* pFsm; SSyncFSM* pFsm;
SMsgCb* msgcb; SMsgCb* msgcb;
int32_t pingMs; int32_t pingMs;

9
include/util/talgo.h
View File

@ -52,7 +52,7 @@ typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void
* @param param * @param param
* @param comparFn * @param comparFn
*/ */
void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); int32_t taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn);
/** /**
* Non-recursive quick sort. * Non-recursive quick sort.
@ -98,8 +98,8 @@ void *taosbsearch(const void *key, const void *base, int32_t nmemb, int32_t size
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, int32_t taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar,
__ext_compar_fn_t compar, char *buf, bool maxroot); __ext_compar_fn_t compar, char *buf, bool maxroot);
/** /**
* sort heap to make sure it is a max/min root heap * sort heap to make sure it is a max/min root heap
@ -114,7 +114,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const
* @param maxroot: if heap is max root heap * @param maxroot: if heap is max root heap
* @return * @return
*/ */
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, bool maxroot); int32_t taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar,
bool maxroot);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
include/util/taoserror.h
View File

@ -224,6 +224,7 @@ int32_t taosGetErrSize();
// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x // #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x
#define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315) #define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315)
#define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316) #define TSDB_CODE_MND_INVALID_PLATFORM TAOS_DEF_ERROR_CODE(0, 0x0316)
#define TSDB_CODE_MND_RETURN_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x0317)
// mnode-sdb // mnode-sdb
#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal #define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal
@ -942,6 +943,11 @@ int32_t taosGetErrSize();
// UTIL // UTIL
#define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000) #define TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x6000)
// AUDIT
#define TSDB_CODE_AUDIT_NOT_FORMAT_TO_JSON TAOS_DEF_ERROR_CODE(0, 0x6100)
#define TSDB_CODE_AUDIT_FAIL_SEND_AUDIT_RECORD TAOS_DEF_ERROR_CODE(0, 0x6101)
#define TSDB_CODE_AUDIT_FAIL_GENERATE_JSON TAOS_DEF_ERROR_CODE(0, 0x6102)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
include/util/tarray.h
View File

@ -251,7 +251,7 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t
* @return * @return
*/ */
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); int32_t taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param);
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz, int8_t sver);

22
include/util/tbloomfilter.h
View File

@ -19,6 +19,8 @@
#include "os.h" #include "os.h"
#include "tencode.h" #include "tencode.h"
#include "thash.h" #include "thash.h"
#include "tlog.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -35,19 +37,19 @@ typedef struct SBloomFilter {
uint64_t size; uint64_t size;
_hash_fn_t hashFn1; _hash_fn_t hashFn1;
_hash_fn_t hashFn2; _hash_fn_t hashFn2;
void *buffer; void* buffer;
double errorRate; double errorRate;
} SBloomFilter; } SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate); int32_t tBloomFilterInit(uint64_t expectedEntries, double errorRate, SBloomFilter** ppBF);
int32_t tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2); int32_t tBloomFilterPutHash(SBloomFilter* pBF, uint64_t hash1, uint64_t hash2);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len); int32_t tBloomFilterPut(SBloomFilter* pBF, const void* keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2); int32_t tBloomFilterNoContain(const SBloomFilter* pBF, uint64_t h1, uint64_t h2);
void tBloomFilterDestroy(SBloomFilter *pBF); void tBloomFilterDestroy(SBloomFilter* pBF);
void tBloomFilterDump(const SBloomFilter *pBF); void tBloomFilterDump(const SBloomFilter* pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF); bool tBloomFilterIsFull(const SBloomFilter* pBF);
int32_t tBloomFilterEncode(const SBloomFilter *pBF, SEncoder *pEncoder); int32_t tBloomFilterEncode(const SBloomFilter* pBF, SEncoder* pEncoder);
SBloomFilter *tBloomFilterDecode(SDecoder *pDecoder); int32_t tBloomFilterDecode(SDecoder* pDecoder, SBloomFilter** ppBF);
#ifdef __cplusplus #ifdef __cplusplus
} }

17
include/util/tbuffer.inc
View File

@ -15,6 +15,7 @@
#include "taoserror.h" #include "taoserror.h"
#include "tcoding.h" #include "tcoding.h"
#include "tutil.h"
struct SBuffer { struct SBuffer {
uint32_t size; uint32_t size;
@ -67,8 +68,7 @@ static FORCE_INLINE int32_t tBufferEnsureCapacity(SBuffer *buffer, uint32_t capa
} }
static FORCE_INLINE int32_t tBufferPut(SBuffer *buffer, const void *data, uint32_t size) { static FORCE_INLINE int32_t tBufferPut(SBuffer *buffer, const void *data, uint32_t size) {
int32_t code = tBufferEnsureCapacity(buffer, buffer->size + size); TAOS_CHECK_RETURN(tBufferEnsureCapacity(buffer, buffer->size + size));
if (code) return code;
memcpy((char *)buffer->data + buffer->size, data, size); memcpy((char *)buffer->data + buffer->size, data, size);
buffer->size += size; buffer->size += size;
return 0; return 0;
@ -119,10 +119,8 @@ static FORCE_INLINE int32_t tBufferPutU16v(SBuffer *buffer, uint16_t value) { re
static FORCE_INLINE int32_t tBufferPutU32v(SBuffer *buffer, uint32_t value) { return tBufferPutU64v(buffer, value); } static FORCE_INLINE int32_t tBufferPutU32v(SBuffer *buffer, uint32_t value) { return tBufferPutU64v(buffer, value); }
static FORCE_INLINE int32_t tBufferPutU64v(SBuffer *buffer, uint64_t value) { static FORCE_INLINE int32_t tBufferPutU64v(SBuffer *buffer, uint64_t value) {
int32_t code;
while (value >= 0x80) { while (value >= 0x80) {
code = tBufferPutU8(buffer, (value & 0x7F) | 0x80); TAOS_CHECK_RETURN(tBufferPutU8(buffer, (value & 0x7F) | 0x80));
if (code) return code;
value >>= 7; value >>= 7;
} }
return tBufferPutU8(buffer, value); return tBufferPutU8(buffer, value);
@ -141,8 +139,7 @@ static FORCE_INLINE int32_t tBufferPutI64v(SBuffer *buffer, int64_t value) {
} }
static FORCE_INLINE int32_t tBufferPutBinary(SBuffer *buffer, const void *data, uint32_t size) { static FORCE_INLINE int32_t tBufferPutBinary(SBuffer *buffer, const void *data, uint32_t size) {
int32_t code = tBufferPutU32v(buffer, size); TAOS_CHECK_RETURN(tBufferPutU32v(buffer, size));
if (code) return code;
return tBufferPut(buffer, data, size); return tBufferPut(buffer, data, size);
} }
@ -324,8 +321,7 @@ static int32_t tBufferGetF32(SBufferReader *reader, float *value) {
float f; float f;
uint32_t u; uint32_t u;
} u; } u;
int32_t code = tBufferGetU32(reader, &u.u); TAOS_CHECK_RETURN(tBufferGetU32(reader, &u.u));
if (code) return code;
if (value) { if (value) {
*value = u.f; *value = u.f;
} }
@ -337,8 +333,7 @@ static int32_t tBufferGetF64(SBufferReader *reader, double *value) {
double f; double f;
uint64_t u; uint64_t u;
} u; } u;
int32_t code = tBufferGetU64(reader, &u.u); TAOS_CHECK_RETURN(tBufferGetU64(reader, &u.u));
if (code) return code;
if (value) { if (value) {
*value = u.f; *value = u.f;
} }

368
include/util/tencode.h
View File

@ -18,6 +18,7 @@
#include "tcoding.h" #include "tcoding.h"
#include "tlist.h" #include "tlist.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -46,38 +47,8 @@ typedef struct {
SDecoderNode* dStack; SDecoderNode* dStack;
} SDecoder; } SDecoder;
#define tPut(TYPE, BUF, VAL) ((TYPE*)(BUF))[0] = (VAL) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define tGet(TYPE, BUF, VAL) (VAL) = ((TYPE*)(BUF))[0] #define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tRPut16(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[0];
#define tRPut32(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[0];
#define tRPut64(PDEST, PSRC) \
((uint8_t*)(PDEST))[0] = ((uint8_t*)(PSRC))[7]; \
((uint8_t*)(PDEST))[1] = ((uint8_t*)(PSRC))[6]; \
((uint8_t*)(PDEST))[2] = ((uint8_t*)(PSRC))[5]; \
((uint8_t*)(PDEST))[3] = ((uint8_t*)(PSRC))[4]; \
((uint8_t*)(PDEST))[4] = ((uint8_t*)(PSRC))[3]; \
((uint8_t*)(PDEST))[5] = ((uint8_t*)(PSRC))[2]; \
((uint8_t*)(PDEST))[6] = ((uint8_t*)(PSRC))[1]; \
((uint8_t*)(PDEST))[7] = ((uint8_t*)(PSRC))[0];
#define tRGet16 tRPut16
#define tRGet32 tRPut32
#define tRGet64 tRPut64
#define TD_CODER_POS(CODER) ((CODER)->pos)
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos)
#define tEncodeSize(E, S, SIZE, RET) \ #define tEncodeSize(E, S, SIZE, RET) \
do { \ do { \
@ -100,6 +71,7 @@ void tEncoderInit(SEncoder* pCoder, uint8_t* data, uint32_t size);
void tEncoderClear(SEncoder* pCoder); void tEncoderClear(SEncoder* pCoder);
int32_t tStartEncode(SEncoder* pCoder); int32_t tStartEncode(SEncoder* pCoder);
void tEndEncode(SEncoder* pCoder); void tEndEncode(SEncoder* pCoder);
static int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size);
static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val); static int32_t tEncodeU8(SEncoder* pCoder, uint8_t val);
static int32_t tEncodeI8(SEncoder* pCoder, int8_t val); static int32_t tEncodeI8(SEncoder* pCoder, int8_t val);
static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val); static int32_t tEncodeU16(SEncoder* pCoder, uint16_t val);
@ -127,6 +99,7 @@ void tDecoderClear(SDecoder* SDecoder);
int32_t tStartDecode(SDecoder* pCoder); int32_t tStartDecode(SDecoder* pCoder);
void tEndDecode(SDecoder* pCoder); void tEndDecode(SDecoder* pCoder);
static bool tDecodeIsEnd(SDecoder* pCoder); static bool tDecodeIsEnd(SDecoder* pCoder);
static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size);
static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val); static int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val);
static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val); static int32_t tDecodeI8(SDecoder* pCoder, int8_t* val);
static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val); static int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val);
@ -149,97 +122,65 @@ static int32_t tDecodeCStr(SDecoder* pCoder, char** val);
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val); static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val);
/* ------------------------ IMPL ------------------------ */ /* ------------------------ IMPL ------------------------ */
#define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ static FORCE_INLINE int32_t tEncodeFixed(SEncoder* pCoder, const void* val, uint32_t size) {
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(VAL))) return -1; \
tPut(TYPE, TD_CODER_CURRENT(CODER), (VAL)); \
} \
TD_CODER_MOVE_POS(CODER, sizeof(VAL)); \
return 0;
#define TD_ENCODE_VARIANT_MACRO(CODER, VAL) \
while ((VAL) >= ENCODE_LIMIT) { \
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = ((VAL) | ENCODE_LIMIT) & 0xff; \
} \
\
(VAL) >>= 7; \
TD_CODER_MOVE_POS(CODER, 1); \
} \
\
if ((CODER)->data) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TD_CODER_CURRENT(CODER)[0] = (uint8_t)(VAL); \
} \
TD_CODER_MOVE_POS(CODER, 1); \
return 0;
#define TD_DECODE_MACRO(CODER, PVAL, TYPE, BITS) \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, sizeof(*(PVAL)))) return -1; \
tGet(TYPE, TD_CODER_CURRENT(CODER), *(PVAL)); \
TD_CODER_MOVE_POS(CODER, sizeof(*(PVAL))); \
return 0;
#define TD_DECODE_VARIANT_MACRO(CODER, PVAL, TYPE) \
int32_t i = 0; \
*(PVAL) = 0; \
for (;;) { \
if (TD_CODER_CHECK_CAPACITY_FAILED(CODER, 1)) return -1; \
TYPE tval = TD_CODER_CURRENT(CODER)[0]; \
if (tval < ENCODE_LIMIT) { \
*(PVAL) |= (tval << (7 * i)); \
TD_CODER_MOVE_POS(pCoder, 1); \
break; \
} else { \
*(PVAL) |= (((tval) & (ENCODE_LIMIT - 1)) << (7 * i)); \
i++; \
TD_CODER_MOVE_POS(pCoder, 1); \
} \
} \
\
return 0;
// 8
static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) {
if (pCoder->data) { if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; if (pCoder->pos + size > pCoder->size) {
tPut(uint8_t, TD_CODER_CURRENT(pCoder), val); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, size);
} }
TD_CODER_MOVE_POS(pCoder, sizeof(val));
pCoder->pos += size;
return 0; return 0;
} }
static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { static FORCE_INLINE int32_t tEncodeU8(SEncoder* pCoder, uint8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
if (pCoder->data) { static FORCE_INLINE int32_t tEncodeI8(SEncoder* pCoder, int8_t val) { return tEncodeFixed(pCoder, &val, sizeof(val)); }
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(val))) return -1; static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) {
tPut(int8_t, TD_CODER_CURRENT(pCoder), val); return tEncodeFixed(pCoder, &val, sizeof(val));
} }
TD_CODER_MOVE_POS(pCoder, sizeof(val)); static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) {
return 0; return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) {
return tEncodeFixed(pCoder, &val, sizeof(val));
}
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) {
while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
} }
// 16
static FORCE_INLINE int32_t tEncodeU16(SEncoder* pCoder, uint16_t val) { TD_ENCODE_MACRO(pCoder, val, uint16_t, 16); }
static FORCE_INLINE int32_t tEncodeI16(SEncoder* pCoder, int16_t val) { TD_ENCODE_MACRO(pCoder, val, int16_t, 16); }
// 32
static FORCE_INLINE int32_t tEncodeU32(SEncoder* pCoder, uint32_t val) { TD_ENCODE_MACRO(pCoder, val, uint32_t, 32); }
static FORCE_INLINE int32_t tEncodeI32(SEncoder* pCoder, int32_t val) { TD_ENCODE_MACRO(pCoder, val, int32_t, 32); }
// 64
static FORCE_INLINE int32_t tEncodeU64(SEncoder* pCoder, uint64_t val) { TD_ENCODE_MACRO(pCoder, val, uint64_t, 64); }
static FORCE_INLINE int32_t tEncodeI64(SEncoder* pCoder, int64_t val) { TD_ENCODE_MACRO(pCoder, val, int64_t, 64); }
// 16v
static FORCE_INLINE int32_t tEncodeU16v(SEncoder* pCoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); }
static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) { static FORCE_INLINE int32_t tEncodeI16v(SEncoder* pCoder, int16_t val) {
return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val)); return tEncodeU16v(pCoder, ZIGZAGE(int16_t, val));
} }
// 32v static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) {
static FORCE_INLINE int32_t tEncodeU32v(SEncoder* pCoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) { static FORCE_INLINE int32_t tEncodeI32v(SEncoder* pCoder, int32_t val) {
return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val)); return tEncodeU32v(pCoder, ZIGZAGE(int32_t, val));
} }
// 64v static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) {
static FORCE_INLINE int32_t tEncodeU64v(SEncoder* pCoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pCoder, val); } while (val >= ENCODE_LIMIT) {
TAOS_CHECK_RETURN(tEncodeU8(pCoder, (val | ENCODE_LIMIT) & 0xff));
val >>= 7;
}
return tEncodeU8(pCoder, val);
}
static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) { static FORCE_INLINE int32_t tEncodeI64v(SEncoder* pCoder, int64_t val) {
return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val)); return tEncodeU64v(pCoder, ZIGZAGE(int64_t, val));
} }
@ -265,14 +206,16 @@ static FORCE_INLINE int32_t tEncodeDouble(SEncoder* pCoder, double val) {
} }
static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) { static FORCE_INLINE int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len) {
if (tEncodeU32v(pCoder, len) < 0) return -1; TAOS_CHECK_RETURN(tEncodeU32v(pCoder, len));
if (len) { if (len) {
if (pCoder->data) { if (pCoder->data) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, len)) return -1; if (pCoder->pos + len > pCoder->size) {
memcpy(TD_CODER_CURRENT(pCoder), val, len); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
memcpy(pCoder->data + pCoder->pos, val, len);
} }
TD_CODER_MOVE_POS(pCoder, len); pCoder->pos += len;
} }
return 0; return 0;
} }
@ -286,70 +229,137 @@ static FORCE_INLINE int32_t tEncodeCStr(SEncoder* pCoder, const char* val) {
} }
/* ------------------------ FOR DECODER ------------------------ */ /* ------------------------ FOR DECODER ------------------------ */
// 8 static int32_t tDecodeFixed(SDecoder* pCoder, void* val, uint32_t size) {
static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) { if (pCoder->pos + size > pCoder->size) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
tGet(uint8_t, TD_CODER_CURRENT(pCoder), *val); } else if (val) {
TD_CODER_MOVE_POS(pCoder, sizeof(*val)); memcpy(val, pCoder->data + pCoder->pos, size);
}
pCoder->pos += size;
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { static FORCE_INLINE int32_t tDecodeU8(SDecoder* pCoder, uint8_t* val) {
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, sizeof(*val))) return -1; return tDecodeFixed(pCoder, val, sizeof(*val));
tGet(int8_t, TD_CODER_CURRENT(pCoder), *val);
TD_CODER_MOVE_POS(pCoder, sizeof(*val));
return 0;
} }
static FORCE_INLINE int32_t tDecodeI8(SDecoder* pCoder, int8_t* val) { return tDecodeFixed(pCoder, val, sizeof(*val)); }
// 16 // 16
static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) { TD_DECODE_MACRO(pCoder, val, uint16_t, 16); } static FORCE_INLINE int32_t tDecodeU16(SDecoder* pCoder, uint16_t* val) {
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) { TD_DECODE_MACRO(pCoder, val, int16_t, 16); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI16(SDecoder* pCoder, int16_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 32 // 32
static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) { TD_DECODE_MACRO(pCoder, val, uint32_t, 32); } static FORCE_INLINE int32_t tDecodeU32(SDecoder* pCoder, uint32_t* val) {
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) { TD_DECODE_MACRO(pCoder, val, int32_t, 32); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI32(SDecoder* pCoder, int32_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 64 // 64
static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) { TD_DECODE_MACRO(pCoder, val, uint64_t, 64); } static FORCE_INLINE int32_t tDecodeU64(SDecoder* pCoder, uint64_t* val) {
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) { TD_DECODE_MACRO(pCoder, val, int64_t, 64); } return tDecodeFixed(pCoder, val, sizeof(*val));
}
static FORCE_INLINE int32_t tDecodeI64(SDecoder* pCoder, int64_t* val) {
return tDecodeFixed(pCoder, val, sizeof(*val));
}
// 16v // 16v
static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) { static FORCE_INLINE int32_t tDecodeU16v(SDecoder* pCoder, uint16_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint16_t); uint8_t byte;
uint16_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint16_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint16_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) { static FORCE_INLINE int32_t tDecodeI16v(SDecoder* pCoder, int16_t* val) {
uint16_t tval; uint16_t tval;
if (tDecodeU16v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU16v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int16_t, tval);
} }
if (val) *val = ZIGZAGD(int16_t, tval);
return 0; return 0;
} }
// 32v // 32v
static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) { static FORCE_INLINE int32_t tDecodeU32v(SDecoder* pCoder, uint32_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint32_t); uint8_t byte;
uint32_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint32_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint32_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) { static FORCE_INLINE int32_t tDecodeI32v(SDecoder* pCoder, int32_t* val) {
uint32_t tval; uint32_t tval;
if (tDecodeU32v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int32_t, tval);
} }
if (val) *val = ZIGZAGD(int32_t, tval);
return 0; return 0;
} }
// 64v // 64v
static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) { static FORCE_INLINE int32_t tDecodeU64v(SDecoder* pCoder, uint64_t* val) {
TD_DECODE_VARIANT_MACRO(pCoder, val, uint64_t); uint8_t byte;
uint64_t tval = 0;
for (int32_t i = 0;; i++) {
TAOS_CHECK_RETURN(tDecodeU8(pCoder, &byte));
if (byte < ENCODE_LIMIT) {
tval |= (((uint64_t)byte) << (7 * i));
break;
} else {
tval |= ((((uint64_t)byte) & (ENCODE_LIMIT - 1)) << (7 * i));
}
}
if (val) {
*val = tval;
}
return 0;
} }
static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) { static FORCE_INLINE int32_t tDecodeI64v(SDecoder* pCoder, int64_t* val) {
uint64_t tval; uint64_t tval;
if (tDecodeU64v(pCoder, &tval) < 0) { TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &tval));
return -1;
if (val) {
*val = ZIGZAGD(int64_t, tval);
} }
if (val) *val = ZIGZAGD(int64_t, tval);
return 0; return 0;
} }
@ -359,11 +369,11 @@ static FORCE_INLINE int32_t tDecodeFloat(SDecoder* pCoder, float* val) {
float f; float f;
} v; } v;
if (tDecodeU32(pCoder, &(v.ui)) < 0) { TAOS_CHECK_RETURN(tDecodeU32(pCoder, &(v.ui)));
return -1;
}
*val = v.f; if (val) {
*val = v.f;
}
return 0; return 0;
} }
@ -373,30 +383,36 @@ static FORCE_INLINE int32_t tDecodeDouble(SDecoder* pCoder, double* val) {
double d; double d;
} v; } v;
if (tDecodeU64(pCoder, &(v.ui)) < 0) { TAOS_CHECK_RETURN(tDecodeU64(pCoder, &(v.ui)));
return -1;
}
*val = v.d; if (val) {
*val = v.d;
}
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeBinary(SDecoder* pCoder, uint8_t** val, uint32_t* len) {
uint32_t length = 0; uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1;
if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (val) { if (len) {
*val = (uint8_t*)TD_CODER_CURRENT(pCoder); *len = length;
} }
TD_CODER_MOVE_POS(pCoder, length); if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
if (val) {
*val = pCoder->data + pCoder->pos;
}
pCoder->pos += length;
return 0; return 0;
} }
static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeCStrAndLen(SDecoder* pCoder, char** val, uint32_t* len) {
if (tDecodeBinary(pCoder, (uint8_t**)val, len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeBinary(pCoder, (uint8_t**)val, len));
(*len) -= 1; (*len) -= 1;
return 0; return 0;
} }
@ -409,7 +425,7 @@ static FORCE_INLINE int32_t tDecodeCStr(SDecoder* pCoder, char** val) {
static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) { static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
char* pStr; char* pStr;
uint32_t len; uint32_t len;
if (tDecodeCStrAndLen(pCoder, &pStr, &len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrAndLen(pCoder, &pStr, &len));
memcpy(val, pStr, len + 1); memcpy(val, pStr, len + 1);
return 0; return 0;
@ -417,16 +433,22 @@ static int32_t tDecodeCStrTo(SDecoder* pCoder, char* val) {
static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) { static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uint64_t* len) {
uint64_t length = 0; uint64_t length = 0;
if (tDecodeU64v(pCoder, &length) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU64v(pCoder, &length));
if (length) { if (length) {
if (len) *len = length; if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; if (pCoder->pos + length > pCoder->size) {
*val = taosMemoryMalloc(length); TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
if (*val == NULL) return -1; }
memcpy(*val, TD_CODER_CURRENT(pCoder), length);
TD_CODER_MOVE_POS(pCoder, length); *val = taosMemoryMalloc(length);
if (*val == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
pCoder->pos += length;
} else { } else {
*val = NULL; *val = NULL;
} }
@ -435,16 +457,20 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc(SDecoder* pCoder, void** val, uin
static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) { static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, uint32_t* len) {
uint32_t length = 0; uint32_t length = 0;
if (tDecodeU32v(pCoder, &length) < 0) return -1; TAOS_CHECK_RETURN(tDecodeU32v(pCoder, &length));
if (length) { if (length) {
if (len) *len = length; if (len) *len = length;
if (TD_CODER_CHECK_CAPACITY_FAILED(pCoder, length)) return -1; if (pCoder->pos + length > pCoder->size) {
TAOS_RETURN(TSDB_CODE_OUT_OF_RANGE);
}
*val = taosMemoryMalloc(length); *val = taosMemoryMalloc(length);
if (*val == NULL) return -1; if (*val == NULL) {
memcpy(*val, TD_CODER_CURRENT(pCoder), length); TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(*val, pCoder->data + pCoder->pos, length);
TD_CODER_MOVE_POS(pCoder, length); pCoder->pos += length;
} else { } else {
*val = NULL; *val = NULL;
} }
@ -452,7 +478,7 @@ static FORCE_INLINE int32_t tDecodeBinaryAlloc32(SDecoder* pCoder, void** val, u
} }
static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) { static FORCE_INLINE int32_t tDecodeCStrAndLenAlloc(SDecoder* pCoder, char** val, uint64_t* len) {
if (tDecodeBinaryAlloc(pCoder, (void**)val, len) < 0) return -1; TAOS_CHECK_RETURN(tDecodeBinaryAlloc(pCoder, (void**)val, len));
(*len) -= 1; (*len) -= 1;
return 0; return 0;
} }

3
include/util/thash.h
View File

@ -28,8 +28,7 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *);
#define HASH_KEY_ALREADY_EXISTS (-2) #define HASH_NODE_EXIST(code) (code == TSDB_CODE_DUP_KEY)
#define HASH_NODE_EXIST(code) (code == HASH_KEY_ALREADY_EXISTS)
/** /**
* murmur hash algorithm * murmur hash algorithm

56
include/util/tqueue.h
View File

@ -73,36 +73,36 @@ struct STaosQnode {
char item[]; char item[];
}; };
STaosQueue *taosOpenQueue(); int32_t taosOpenQueue(STaosQueue **queue);
void taosCloseQueue(STaosQueue *queue); void taosCloseQueue(STaosQueue *queue);
void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp);
void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize); int32_t taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize, void **item);
void taosFreeQitem(void *pItem); void taosFreeQitem(void *pItem);
int32_t taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem);
int32_t taosReadQitem(STaosQueue *queue, void **ppItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem);
bool taosQueueEmpty(STaosQueue *queue); bool taosQueueEmpty(STaosQueue *queue);
void taosUpdateItemSize(STaosQueue *queue, int32_t items); void taosUpdateItemSize(STaosQueue *queue, int32_t items);
int32_t taosQueueItemSize(STaosQueue *queue); int32_t taosQueueItemSize(STaosQueue *queue);
int64_t taosQueueMemorySize(STaosQueue *queue); int64_t taosQueueMemorySize(STaosQueue *queue);
void taosSetQueueCapacity(STaosQueue *queue, int64_t size); void taosSetQueueCapacity(STaosQueue *queue, int64_t size);
void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t mem); void taosSetQueueMemoryCapacity(STaosQueue *queue, int64_t mem);
STaosQall *taosAllocateQall(); int32_t taosAllocateQall(STaosQall **qall);
void taosFreeQall(STaosQall *qall); void taosFreeQall(STaosQall *qall);
int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall); int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
int32_t taosGetQitem(STaosQall *qall, void **ppItem); int32_t taosGetQitem(STaosQall *qall, void **ppItem);
void taosResetQitems(STaosQall *qall); void taosResetQitems(STaosQall *qall);
int32_t taosQallItemSize(STaosQall *qall); int32_t taosQallItemSize(STaosQall *qall);
int64_t taosQallMemSize(STaosQall *qll); int64_t taosQallMemSize(STaosQall *qll);
int64_t taosQallUnAccessedItemSize(STaosQall *qall); int64_t taosQallUnAccessedItemSize(STaosQall *qall);
int64_t taosQallUnAccessedMemSize(STaosQall *qall); int64_t taosQallUnAccessedMemSize(STaosQall *qall);
STaosQset *taosOpenQset(); int32_t taosOpenQset(STaosQset **qset);
void taosCloseQset(STaosQset *qset); void taosCloseQset(STaosQset *qset);
void taosQsetThreadResume(STaosQset *qset); void taosQsetThreadResume(STaosQset *qset);
int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle);
void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue);
int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosGetQueueNumber(STaosQset *qset);
int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo);
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);

24
include/util/tscalablebf.h
View File

@ -23,22 +23,22 @@ extern "C" {
#endif #endif
typedef struct SScalableBf { typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters SArray* bfArray; // array of bloom filters
uint32_t growth; uint32_t growth;
uint64_t numBits; uint64_t numBits;
uint32_t maxBloomFilters; uint32_t maxBloomFilters;
int8_t status; int8_t status;
_hash_fn_t hashFn1; _hash_fn_t hashFn1;
_hash_fn_t hashFn2; _hash_fn_t hashFn2;
} SScalableBf; } SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate); int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf);
int32_t tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len); int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf); void tScalableBfDestroy(SScalableBf* pSBf);
int32_t tScalableBfEncode(const SScalableBf *pSBf, SEncoder *pEncoder); int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder);
SScalableBf *tScalableBfDecode(SDecoder *pDecoder); int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf);
#ifdef __cplusplus #ifdef __cplusplus
} }

3
include/util/tutil.h
View File

@ -21,6 +21,7 @@
#include "tdef.h" #include "tdef.h"
#include "thash.h" #include "thash.h"
#include "tmd5.h" #include "tmd5.h"
#include "tutil.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -171,6 +172,8 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
} \ } \
} while (0) } while (0)
#define TAOS_UNUSED(expr) (void)(expr)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

18
source/client/inc/clientSml.h
View File

@ -82,7 +82,7 @@ extern "C" {
#define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \ #define IS_SLASH_LETTER_IN_MEASUREMENT(sql) \
(*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH)) (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == SLASH))
#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) #define MOVE_FORWARD_ONE(sql, len) ((void)memmove((void *)((sql)-1), (sql), len))
#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \ #define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen) \
for (int i = 1; i < keyLen; ++i) { \ for (int i = 1; i < keyLen; ++i) { \
@ -211,19 +211,19 @@ extern int64_t smlFactorS[];
typedef int32_t (*_equal_fn_sml)(const void *, const void *); typedef int32_t (*_equal_fn_sml)(const void *, const void *);
SSmlHandle *smlBuildSmlInfo(TAOS *taos); int32_t smlBuildSmlInfo(TAOS *taos, SSmlHandle **handle);
void smlDestroyInfo(SSmlHandle *info); void smlDestroyInfo(SSmlHandle *info);
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); void smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); int32_t smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); int32_t smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen, SSmlTableInfo** tInfo);
SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); int32_t smlBuildSTableMeta(bool isDataFormat, SSmlSTableMeta** sMeta);
int32_t smlSetCTableName(SSmlTableInfo *oneTable); int32_t smlSetCTableName(SSmlTableInfo *oneTable);
void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo); int32_t getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tinfo);
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); int32_t smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen, STableMeta **pTableMeta);
int32_t is_same_child_table_telnet(const void *a, const void *b); int32_t is_same_child_table_telnet(const void *a, const void *b);
int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len);
int32_t smlClearForRerun(SSmlHandle *info); int32_t smlClearForRerun(SSmlHandle *info);
@ -236,7 +236,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
int32_t smlParseJSON(SSmlHandle *info, char *payload); int32_t smlParseJSON(SSmlHandle *info, char *payload);
SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement); int32_t smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement, SSmlSTableMeta** sMeta);
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv); bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements); int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);

2
source/client/src/clientHb.c
View File

@ -744,7 +744,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
} }
if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) != 0) {
taosMemoryFree(user); taosMemoryFree(user);
code = terrno ? terrno : TSDB_CODE_APP_ERROR; code = terrno ? terrno : TSDB_CODE_APP_ERROR;
goto _return; goto _return;

350
source/client/src/clientMonitor.c
View File

@ -1,13 +1,13 @@
#include "clientMonitor.h" #include "clientMonitor.h"
#include "clientLog.h"
#include "os.h"
#include "tmisce.h"
#include "ttime.h"
#include "ttimer.h"
#include "tglobal.h"
#include "tqueue.h"
#include "cJSON.h" #include "cJSON.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "tglobal.h"
#include "tmisce.h"
#include "tqueue.h"
#include "ttime.h"
#include "ttimer.h"
SRWLatch monitorLock; SRWLatch monitorLock;
void* monitorTimer; void* monitorTimer;
@ -20,41 +20,41 @@ STaosQueue* monitorQueue;
SHashObj* monitorSlowLogHash; SHashObj* monitorSlowLogHash;
char tmpSlowLogPath[PATH_MAX] = {0}; char tmpSlowLogPath[PATH_MAX] = {0};
static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size){ static int32_t getSlowLogTmpDir(char* tmpPath, int32_t size) {
if (tsTempDir == NULL) { if (tsTempDir == NULL) {
return -1; return -1;
} }
int ret = snprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir); int ret = snprintf(tmpPath, size, "%s/tdengine_slow_log/", tsTempDir);
if (ret < 0){ if (ret < 0) {
tscError("failed to get tmp path ret:%d", ret); tscError("failed to get tmp path ret:%d", ret);
return ret; return ret;
} }
return 0; return 0;
} }
static void processFileInTheEnd(TdFilePtr pFile, char* path){ static void processFileInTheEnd(TdFilePtr pFile, char* path) {
if(pFile == NULL){ if (pFile == NULL) {
return; return;
} }
if(taosFtruncateFile(pFile, 0) != 0){ if (taosFtruncateFile(pFile, 0) != 0) {
tscError("failed to truncate file:%s, errno:%d", path, errno); tscError("failed to truncate file:%s, errno:%d", path, errno);
return; return;
} }
if(taosUnLockFile(pFile) != 0){ if (taosUnLockFile(pFile) != 0) {
tscError("failed to unlock file:%s, errno:%d", path, errno); tscError("failed to unlock file:%s, errno:%d", path, errno);
return; return;
} }
if(taosCloseFile(&(pFile)) != 0){ if (taosCloseFile(&(pFile)) != 0) {
tscError("failed to close file:%s, errno:%d", path, errno); tscError("failed to close file:%s, errno:%d", path, errno);
return; return;
} }
if(taosRemoveFile(path) != 0){ if (taosRemoveFile(path) != 0) {
tscError("failed to remove file:%s, errno:%d", path, errno); tscError("failed to remove file:%s, errno:%d", path, errno);
return; return;
} }
} }
static void destroySlowLogClient(void* data){ static void destroySlowLogClient(void* data) {
if (data == NULL) { if (data == NULL) {
return; return;
} }
@ -63,40 +63,40 @@ static void destroySlowLogClient(void* data){
taosMemoryFree(slowLogClient); taosMemoryFree(slowLogClient);
} }
static void destroyMonitorClient(void* data){ static void destroyMonitorClient(void* data) {
if (data == NULL) { if (data == NULL) {
return; return;
} }
MonitorClient* pMonitor = *(MonitorClient**)data; MonitorClient* pMonitor = *(MonitorClient**)data;
if(pMonitor == NULL){ if (pMonitor == NULL) {
return; return;
} }
taosTmrStopA(&pMonitor->timer); taosTmrStopA(&pMonitor->timer);
taosHashCleanup(pMonitor->counters); taosHashCleanup(pMonitor->counters);
taos_collector_registry_destroy(pMonitor->registry); taos_collector_registry_destroy(pMonitor->registry);
// taos_collector_destroy(pMonitor->colector); // taos_collector_destroy(pMonitor->colector);
taosMemoryFree(pMonitor); taosMemoryFree(pMonitor);
} }
static void monitorFreeSlowLogData(void *paras) { static void monitorFreeSlowLogData(void* paras) {
MonitorSlowLogData* pData = (MonitorSlowLogData*)paras; MonitorSlowLogData* pData = (MonitorSlowLogData*)paras;
if (pData == NULL) { if (pData == NULL) {
return; return;
} }
taosMemoryFreeClear(pData->data); taosMemoryFreeClear(pData->data);
if (pData->type == SLOW_LOG_READ_BEGINNIG){ if (pData->type == SLOW_LOG_READ_BEGINNIG) {
taosMemoryFree(pData->fileName); taosMemoryFree(pData->fileName);
} }
} }
static void monitorFreeSlowLogDataEx(void *paras) { static void monitorFreeSlowLogDataEx(void* paras) {
monitorFreeSlowLogData(paras); monitorFreeSlowLogData(paras);
taosMemoryFree(paras); taosMemoryFree(paras);
} }
static SAppInstInfo* getAppInstByClusterId(int64_t clusterId) { static SAppInstInfo* getAppInstByClusterId(int64_t clusterId) {
void *p = taosHashGet(appInfo.pInstMapByClusterId, &clusterId, LONG_BYTES); void* p = taosHashGet(appInfo.pInstMapByClusterId, &clusterId, LONG_BYTES);
if(p == NULL){ if (p == NULL) {
tscError("failed to get app inst, clusterId:%" PRIx64, clusterId); tscError("failed to get app inst, clusterId:%" PRIx64, clusterId);
return NULL; return NULL;
} }
@ -111,21 +111,25 @@ static int32_t monitorReportAsyncCB(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
} }
if(param != NULL){ if (param != NULL) {
MonitorSlowLogData* p = (MonitorSlowLogData*)param; MonitorSlowLogData* p = (MonitorSlowLogData*)param;
if(code != 0){ if (code != 0) {
tscError("failed to send slow log:%s, clusterId:%" PRIx64, p->data, p->clusterId); tscError("failed to send slow log:%s, clusterId:%" PRIx64, p->data, p->clusterId);
} }
MonitorSlowLogData tmp = {.clusterId = p->clusterId, .type = p->type, .fileName = p->fileName, MonitorSlowLogData tmp = {.clusterId = p->clusterId,
.pFile= p->pFile, .offset = p->offset, .data = NULL}; .type = p->type,
if(monitorPutData2MonitorQueue(tmp) == 0){ .fileName = p->fileName,
.pFile = p->pFile,
.offset = p->offset,
.data = NULL};
if (monitorPutData2MonitorQueue(tmp) == 0) {
p->fileName = NULL; p->fileName = NULL;
} }
} }
return code; return code;
} }
static int32_t sendReport(void* pTransporter, SEpSet *epSet, char* pCont, MONITOR_TYPE type, void* param) { static int32_t sendReport(void* pTransporter, SEpSet* epSet, char* pCont, MONITOR_TYPE type, void* param) {
SStatisReq sStatisReq; SStatisReq sStatisReq;
sStatisReq.pCont = pCont; sStatisReq.pCont = pCont;
sStatisReq.contLen = strlen(pCont); sStatisReq.contLen = strlen(pCont);
@ -167,11 +171,11 @@ FAILED:
return -1; return -1;
} }
static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet *epSet) { static void generateClusterReport(taos_collector_registry_t* registry, void* pTransporter, SEpSet* epSet) {
char ts[50] = {0}; char ts[50] = {0};
sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); sprintf(ts, "%" PRId64, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL); char* pCont = (char*)taos_collector_registry_bridge_new(registry, ts, "%" PRId64, NULL);
if(NULL == pCont) { if (NULL == pCont) {
tscError("generateClusterReport failed, get null content."); tscError("generateClusterReport failed, get null content.");
return; return;
} }
@ -190,8 +194,8 @@ static void reportSendProcess(void* param, void* tmrId) {
} }
MonitorClient* pMonitor = (MonitorClient*)param; MonitorClient* pMonitor = (MonitorClient*)param;
SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId); SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId);
if(pInst == NULL){ if (pInst == NULL) {
taosRUnLockLatch(&monitorLock); taosRUnLockLatch(&monitorLock);
return; return;
} }
@ -202,15 +206,15 @@ static void reportSendProcess(void* param, void* tmrId) {
taosRUnLockLatch(&monitorLock); taosRUnLockLatch(&monitorLock);
} }
static void sendAllCounter(){ static void sendAllCounter() {
MonitorClient** ppMonitor = NULL; MonitorClient** ppMonitor = NULL;
while ((ppMonitor = taosHashIterate(monitorSlowLogHash, ppMonitor))) { while ((ppMonitor = taosHashIterate(monitorSlowLogHash, ppMonitor))) {
MonitorClient* pMonitor = *ppMonitor; MonitorClient* pMonitor = *ppMonitor;
if (pMonitor == NULL){ if (pMonitor == NULL) {
continue; continue;
} }
SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId); SAppInstInfo* pInst = getAppInstByClusterId(pMonitor->clusterId);
if(pInst == NULL){ if (pInst == NULL) {
taosHashCancelIterate(monitorCounterHash, ppMonitor); taosHashCancelIterate(monitorCounterHash, ppMonitor);
break; break;
} }
@ -231,46 +235,48 @@ void monitorCreateClient(int64_t clusterId) {
} }
pMonitor->clusterId = clusterId; pMonitor->clusterId = clusterId;
char clusterKey[32] = {0}; char clusterKey[32] = {0};
if(snprintf(clusterKey, sizeof(clusterKey), "%"PRId64, clusterId) < 0){ if (snprintf(clusterKey, sizeof(clusterKey), "%" PRId64, clusterId) < 0) {
tscError("failed to create cluster key"); tscError("failed to create cluster key");
goto fail; goto fail;
} }
pMonitor->registry = taos_collector_registry_new(clusterKey); pMonitor->registry = taos_collector_registry_new(clusterKey);
if(pMonitor->registry == NULL){ if (pMonitor->registry == NULL) {
tscError("failed to create registry"); tscError("failed to create registry");
goto fail; goto fail;
} }
pMonitor->colector = taos_collector_new(clusterKey); pMonitor->colector = taos_collector_new(clusterKey);
if(pMonitor->colector == NULL){ if (pMonitor->colector == NULL) {
tscError("failed to create collector"); tscError("failed to create collector");
goto fail; goto fail;
} }
taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector); taos_collector_registry_register_collector(pMonitor->registry, pMonitor->colector);
pMonitor->counters = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pMonitor->counters =
(SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pMonitor->counters == NULL) { if (pMonitor->counters == NULL) {
tscError("failed to create monitor counters"); tscError("failed to create monitor counters");
goto fail; goto fail;
} }
// taosHashSetFreeFp(pMonitor->counters, destroyCounter); // taosHashSetFreeFp(pMonitor->counters, destroyCounter);
if(taosHashPut(monitorCounterHash, &clusterId, LONG_BYTES, &pMonitor, POINTER_BYTES) != 0){ if (taosHashPut(monitorCounterHash, &clusterId, LONG_BYTES, &pMonitor, POINTER_BYTES) != 0) {
tscError("failed to put monitor client to hash"); tscError("failed to put monitor client to hash");
goto fail; goto fail;
} }
SAppInstInfo* pInst = getAppInstByClusterId(clusterId); SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
if(pInst == NULL){ if (pInst == NULL) {
tscError("failed to get app instance by cluster id"); tscError("failed to get app instance by cluster id");
pMonitor = NULL; pMonitor = NULL;
goto fail; goto fail;
} }
pMonitor->timer = taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer); pMonitor->timer =
if(pMonitor->timer == NULL){ taosTmrStart(reportSendProcess, pInst->monitorParas.tsMonitorInterval * 1000, (void*)pMonitor, monitorTimer);
if (pMonitor->timer == NULL) {
tscError("failed to start timer"); tscError("failed to start timer");
goto fail; goto fail;
} }
tscInfo("[monitor] monitorCreateClient for %"PRIx64 "finished %p.", clusterId, pMonitor); tscInfo("[monitor] monitorCreateClient for %" PRIx64 "finished %p.", clusterId, pMonitor);
} }
taosWUnLockLatch(&monitorLock); taosWUnLockLatch(&monitorLock);
if (-1 != atomic_val_compare_exchange_32(&monitorFlag, -1, 0)) { if (-1 != atomic_val_compare_exchange_32(&monitorFlag, -1, 0)) {
@ -283,7 +289,8 @@ fail:
taosWUnLockLatch(&monitorLock); taosWUnLockLatch(&monitorLock);
} }
void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count, const char** label_keys) { void monitorCreateClientCounter(int64_t clusterId, const char* name, const char* help, size_t label_key_count,
const char** label_keys) {
taosWLockLatch(&monitorLock); taosWLockLatch(&monitorLock);
MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES); MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES);
if (ppMonitor == NULL || *ppMonitor == NULL) { if (ppMonitor == NULL || *ppMonitor == NULL) {
@ -291,16 +298,16 @@ void monitorCreateClientCounter(int64_t clusterId, const char* name, const char*
goto end; goto end;
} }
taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys); taos_counter_t* newCounter = taos_counter_new(name, help, label_key_count, label_keys);
if (newCounter == NULL) if (newCounter == NULL) return;
return; MonitorClient* pMonitor = *ppMonitor;
MonitorClient* pMonitor = *ppMonitor;
taos_collector_add_metric(pMonitor->colector, newCounter); taos_collector_add_metric(pMonitor->colector, newCounter);
if(taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0){ if (taosHashPut(pMonitor->counters, name, strlen(name), &newCounter, POINTER_BYTES) != 0) {
tscError("failed to put counter to monitor"); tscError("failed to put counter to monitor");
taos_counter_destroy(newCounter); taos_counter_destroy(newCounter);
goto end; goto end;
} }
tscInfo("[monitor] monitorCreateClientCounter %"PRIx64"(%p):%s : %p.", pMonitor->clusterId, pMonitor, name, newCounter); tscInfo("[monitor] monitorCreateClientCounter %" PRIx64 "(%p):%s : %p.", pMonitor->clusterId, pMonitor, name,
newCounter);
end: end:
taosWUnLockLatch(&monitorLock); taosWUnLockLatch(&monitorLock);
@ -315,18 +322,18 @@ void monitorCounterInc(int64_t clusterId, const char* counterName, const char**
MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES); MonitorClient** ppMonitor = (MonitorClient**)taosHashGet(monitorCounterHash, &clusterId, LONG_BYTES);
if (ppMonitor == NULL || *ppMonitor == NULL) { if (ppMonitor == NULL || *ppMonitor == NULL) {
tscError("monitorCounterInc not found pMonitor %"PRId64, clusterId); tscError("monitorCounterInc not found pMonitor %" PRId64, clusterId);
goto end; goto end;
} }
MonitorClient* pMonitor = *ppMonitor; MonitorClient* pMonitor = *ppMonitor;
taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName)); taos_counter_t** ppCounter = (taos_counter_t**)taosHashGet(pMonitor->counters, counterName, strlen(counterName));
if (ppCounter == NULL || *ppCounter == NULL) { if (ppCounter == NULL || *ppCounter == NULL) {
tscError("monitorCounterInc not found pCounter %"PRIx64":%s.", clusterId, counterName); tscError("monitorCounterInc not found pCounter %" PRIx64 ":%s.", clusterId, counterName);
goto end; goto end;
} }
taos_counter_inc(*ppCounter, label_values); taos_counter_inc(*ppCounter, label_values);
tscDebug("[monitor] monitorCounterInc %"PRIx64"(%p):%s", pMonitor->clusterId, pMonitor, counterName); tscDebug("[monitor] monitorCounterInc %" PRIx64 "(%p):%s", pMonitor->clusterId, pMonitor, counterName);
end: end:
taosWUnLockLatch(&monitorLock); taosWUnLockLatch(&monitorLock);
@ -339,13 +346,13 @@ const char* monitorResultStr(SQL_RESULT_CODE code) {
static void monitorThreadFuncUnexpectedStopped(void) { atomic_store_32(&slowLogFlag, -1); } static void monitorThreadFuncUnexpectedStopped(void) { atomic_store_32(&slowLogFlag, -1); }
static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpPath){ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char* tmpPath) {
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
void* tmp = taosHashGet(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES); void* tmp = taosHashGet(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES);
if (tmp == NULL){ if (tmp == NULL) {
char path[PATH_MAX] = {0}; char path[PATH_MAX] = {0};
char clusterId[32] = {0}; char clusterId[32] = {0};
if (snprintf(clusterId, sizeof(clusterId), "%" PRIx64, slowLogData->clusterId) < 0){ if (snprintf(clusterId, sizeof(clusterId), "%" PRIx64, slowLogData->clusterId) < 0) {
tscError("failed to generate clusterId:%" PRIx64, slowLogData->clusterId); tscError("failed to generate clusterId:%" PRIx64, slowLogData->clusterId);
return; return;
} }
@ -358,8 +365,8 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpP
return; return;
} }
SlowLogClient *pClient = taosMemoryCalloc(1, sizeof(SlowLogClient)); SlowLogClient* pClient = taosMemoryCalloc(1, sizeof(SlowLogClient));
if (pClient == NULL){ if (pClient == NULL) {
tscError("failed to allocate memory for slow log client"); tscError("failed to allocate memory for slow log client");
taosCloseFile(&pFile); taosCloseFile(&pFile);
return; return;
@ -368,58 +375,58 @@ static void monitorWriteSlowLog2File(MonitorSlowLogData* slowLogData, char *tmpP
strcpy(pClient->path, path); strcpy(pClient->path, path);
pClient->offset = 0; pClient->offset = 0;
pClient->pFile = pFile; pClient->pFile = pFile;
if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0){ if (taosHashPut(monitorSlowLogHash, &slowLogData->clusterId, LONG_BYTES, &pClient, POINTER_BYTES) != 0) {
tscError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId); tscError("failed to put clusterId:%" PRId64 " to hash table", slowLogData->clusterId);
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(pClient); taosMemoryFree(pClient);
return; return;
} }
if(taosLockFile(pFile) < 0){ if (taosLockFile(pFile) < 0) {
tscError("failed to lock file:%p since %s", pFile, terrstr()); tscError("failed to lock file:%p since %s", pFile, terrstr());
return; return;
} }
}else{ } else {
pFile = (*(SlowLogClient**)tmp)->pFile; pFile = (*(SlowLogClient**)tmp)->pFile;
} }
if(taosLSeekFile(pFile, 0, SEEK_END) < 0){ if (taosLSeekFile(pFile, 0, SEEK_END) < 0) {
tscError("failed to seek file:%p code: %d", pFile, errno); tscError("failed to seek file:%p code: %d", pFile, errno);
return; return;
} }
if (taosWriteFile(pFile, slowLogData->data, strlen(slowLogData->data) + 1) < 0){ if (taosWriteFile(pFile, slowLogData->data, strlen(slowLogData->data) + 1) < 0) {
tscError("failed to write len to file:%p since %s", pFile, terrstr()); tscError("failed to write len to file:%p since %s", pFile, terrstr());
} }
tscDebug("[monitor] write slow log to file:%p, clusterId:%"PRIx64, pFile, slowLogData->clusterId); tscDebug("[monitor] write slow log to file:%p, clusterId:%" PRIx64, pFile, slowLogData->clusterId);
} }
static char* readFile(TdFilePtr pFile, int64_t *offset, int64_t size){ static char* readFile(TdFilePtr pFile, int64_t* offset, int64_t size) {
tscDebug("[monitor] readFile slow begin pFile:%p, offset:%"PRId64 ", size:%"PRId64, pFile, *offset, size); tscDebug("[monitor] readFile slow begin pFile:%p, offset:%" PRId64 ", size:%" PRId64, pFile, *offset, size);
if(taosLSeekFile(pFile, *offset, SEEK_SET) < 0){ if (taosLSeekFile(pFile, *offset, SEEK_SET) < 0) {
tscError("failed to seek file:%p code: %d", pFile, errno); tscError("failed to seek file:%p code: %d", pFile, errno);
return NULL; return NULL;
} }
ASSERT(size > *offset); ASSERT(size > *offset);
char* pCont = NULL; char* pCont = NULL;
int64_t totalSize = 0; int64_t totalSize = 0;
if (size - *offset >= SLOW_LOG_SEND_SIZE_MAX) { if (size - *offset >= SLOW_LOG_SEND_SIZE_MAX) {
pCont = taosMemoryCalloc(1, 4 + SLOW_LOG_SEND_SIZE_MAX); //4 reserved for [] pCont = taosMemoryCalloc(1, 4 + SLOW_LOG_SEND_SIZE_MAX); // 4 reserved for []
totalSize = 4 + SLOW_LOG_SEND_SIZE_MAX; totalSize = 4 + SLOW_LOG_SEND_SIZE_MAX;
}else{ } else {
pCont = taosMemoryCalloc(1, 4 + (size - *offset)); pCont = taosMemoryCalloc(1, 4 + (size - *offset));
totalSize = 4 + (size - *offset); totalSize = 4 + (size - *offset);
} }
if(pCont == NULL){ if (pCont == NULL) {
tscError("failed to allocate memory for slow log, size:%" PRId64, totalSize); tscError("failed to allocate memory for slow log, size:%" PRId64, totalSize);
return NULL; return NULL;
} }
char* buf = pCont; char* buf = pCont;
strcat(buf++, "["); strcat(buf++, "[");
int64_t readSize = taosReadFile(pFile, buf, SLOW_LOG_SEND_SIZE_MAX); int64_t readSize = taosReadFile(pFile, buf, SLOW_LOG_SEND_SIZE_MAX);
if (readSize <= 0) { if (readSize <= 0) {
if (readSize < 0){ if (readSize < 0) {
tscError("failed to read len from file:%p since %s", pFile, terrstr()); tscError("failed to read len from file:%p since %s", pFile, terrstr());
} }
taosMemoryFree(pCont); taosMemoryFree(pCont);
@ -427,24 +434,24 @@ static char* readFile(TdFilePtr pFile, int64_t *offset, int64_t size){
} }
totalSize = 0; totalSize = 0;
while(1){ while (1) {
size_t len = strlen(buf); size_t len = strlen(buf);
totalSize += (len+1); totalSize += (len + 1);
if (totalSize > readSize || len == 0) { if (totalSize > readSize || len == 0) {
*(buf-1) = ']'; *(buf - 1) = ']';
*buf = '\0'; *buf = '\0';
break; break;
} }
buf[len] = ','; // replace '\0' with ',' buf[len] = ','; // replace '\0' with ','
buf += (len + 1); buf += (len + 1);
*offset += (len+1); *offset += (len + 1);
} }
tscDebug("[monitor] readFile slow log end, data:%s, offset:%"PRId64, pCont, *offset); tscDebug("[monitor] readFile slow log end, data:%s, offset:%" PRId64, pCont, *offset);
return pCont; return pCont;
} }
static int64_t getFileSize(char* path){ static int64_t getFileSize(char* path) {
int64_t fileSize = 0; int64_t fileSize = 0;
if (taosStatFile(path, &fileSize, NULL, NULL) < 0) { if (taosStatFile(path, &fileSize, NULL, NULL) < 0) {
return -1; return -1;
@ -453,13 +460,14 @@ static int64_t getFileSize(char* path){
return fileSize; return fileSize;
} }
static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type, char* fileName, void* pTransporter, SEpSet *epSet){ static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64_t offset, SLOW_LOG_QUEUE_TYPE type,
if (data == NULL){ char* fileName, void* pTransporter, SEpSet* epSet) {
if (data == NULL) {
taosMemoryFree(fileName); taosMemoryFree(fileName);
return -1; return -1;
} }
MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData)); MonitorSlowLogData* pParam = taosMemoryMalloc(sizeof(MonitorSlowLogData));
if(pParam == NULL){ if (pParam == NULL) {
taosMemoryFree(data); taosMemoryFree(data);
taosMemoryFree(fileName); taosMemoryFree(fileName);
return -1; return -1;
@ -473,121 +481,124 @@ static int32_t sendSlowLog(int64_t clusterId, char* data, TdFilePtr pFile, int64
return sendReport(pTransporter, epSet, data, MONITOR_TYPE_SLOW_LOG, pParam); return sendReport(pTransporter, epSet, data, MONITOR_TYPE_SLOW_LOG, pParam);
} }
static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offset, int64_t size, SLOW_LOG_QUEUE_TYPE type, char* fileName){ static int32_t monitorReadSend(int64_t clusterId, TdFilePtr pFile, int64_t* offset, int64_t size,
SLOW_LOG_QUEUE_TYPE type, char* fileName) {
SAppInstInfo* pInst = getAppInstByClusterId(clusterId); SAppInstInfo* pInst = getAppInstByClusterId(clusterId);
if(pInst == NULL){ if (pInst == NULL) {
tscError("failed to get app instance by clusterId:%" PRId64, clusterId); tscError("failed to get app instance by clusterId:%" PRId64, clusterId);
return -1; return -1;
} }
SEpSet ep = getEpSet_s(&pInst->mgmtEp); SEpSet ep = getEpSet_s(&pInst->mgmtEp);
char* data = readFile(pFile, offset, size); char* data = readFile(pFile, offset, size);
return sendSlowLog(clusterId, data, (type == SLOW_LOG_READ_BEGINNIG ? pFile : NULL), *offset, type, fileName, pInst->pTransporter, &ep); return sendSlowLog(clusterId, data, (type == SLOW_LOG_READ_BEGINNIG ? pFile : NULL), *offset, type, fileName,
pInst->pTransporter, &ep);
} }
static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset){ static void monitorSendSlowLogAtBeginning(int64_t clusterId, char** fileName, TdFilePtr pFile, int64_t offset) {
int64_t size = getFileSize(*fileName); int64_t size = getFileSize(*fileName);
if(size <= offset){ if (size <= offset) {
processFileInTheEnd(pFile, *fileName); processFileInTheEnd(pFile, *fileName);
tscDebug("[monitor] monitorSendSlowLogAtBeginning delete file:%s", *fileName); tscDebug("[monitor] monitorSendSlowLogAtBeginning delete file:%s", *fileName);
}else{ } else {
int32_t code = monitorReadSend(clusterId, pFile, &offset, size, SLOW_LOG_READ_BEGINNIG, *fileName); int32_t code = monitorReadSend(clusterId, pFile, &offset, size, SLOW_LOG_READ_BEGINNIG, *fileName);
tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log clusterId:%"PRId64",ret:%d", clusterId, code); tscDebug("[monitor] monitorSendSlowLogAtBeginning send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code);
*fileName = NULL; *fileName = NULL;
} }
} }
static void monitorSendSlowLogAtRunning(int64_t clusterId){ static void monitorSendSlowLogAtRunning(int64_t clusterId) {
void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES); void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES);
if (tmp == NULL){ if (tmp == NULL) {
return; return;
} }
SlowLogClient* pClient = (*(SlowLogClient**)tmp); SlowLogClient* pClient = (*(SlowLogClient**)tmp);
if (pClient == NULL){ if (pClient == NULL) {
return; return;
} }
int64_t size = getFileSize(pClient->path); int64_t size = getFileSize(pClient->path);
if(size <= pClient->offset){ if (size <= pClient->offset) {
if(taosFtruncateFile(pClient->pFile, 0) < 0){ if (taosFtruncateFile(pClient->pFile, 0) < 0) {
tscError("failed to truncate file:%p code: %d", pClient->pFile, errno); tscError("failed to truncate file:%p code: %d", pClient->pFile, errno);
} }
tscDebug("[monitor] monitorSendSlowLogAtRunning truncate file to 0 file:%p", pClient->pFile); tscDebug("[monitor] monitorSendSlowLogAtRunning truncate file to 0 file:%p", pClient->pFile);
pClient->offset = 0; pClient->offset = 0;
}else{ } else {
int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL); int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL);
tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log clusterId:%"PRId64",ret:%d", clusterId, code); tscDebug("[monitor] monitorSendSlowLogAtRunning send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code);
} }
} }
static bool monitorSendSlowLogAtQuit(int64_t clusterId) { static bool monitorSendSlowLogAtQuit(int64_t clusterId) {
void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES); void* tmp = taosHashGet(monitorSlowLogHash, &clusterId, LONG_BYTES);
if (tmp == NULL){ if (tmp == NULL) {
return true; return true;
} }
SlowLogClient* pClient = (*(SlowLogClient**)tmp); SlowLogClient* pClient = (*(SlowLogClient**)tmp);
if (pClient == NULL){ if (pClient == NULL) {
return true; return true;
} }
int64_t size = getFileSize(pClient->path); int64_t size = getFileSize(pClient->path);
if(size <= pClient->offset){ if (size <= pClient->offset) {
processFileInTheEnd(pClient->pFile, pClient->path); processFileInTheEnd(pClient->pFile, pClient->path);
pClient->pFile = NULL; pClient->pFile = NULL;
tscInfo("[monitor] monitorSendSlowLogAtQuit remove file:%s", pClient->path); tscInfo("[monitor] monitorSendSlowLogAtQuit remove file:%s", pClient->path);
if((--quitCnt) == 0){ if ((--quitCnt) == 0) {
return true; return true;
} }
}else{ } else {
int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL); int32_t code = monitorReadSend(clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL);
tscDebug("[monitor] monitorSendSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", clusterId, code); tscDebug("[monitor] monitorSendSlowLogAtQuit send slow log clusterId:%" PRId64 ",ret:%d", clusterId, code);
} }
return false; return false;
} }
static void monitorSendAllSlowLogAtQuit(){ static void monitorSendAllSlowLogAtQuit() {
void* pIter = NULL; void* pIter = NULL;
while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) { while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) {
SlowLogClient* pClient = (*(SlowLogClient**)pIter); SlowLogClient* pClient = (*(SlowLogClient**)pIter);
if(pClient == NULL) { if (pClient == NULL) {
continue; continue;
} }
int64_t size = getFileSize(pClient->path); int64_t size = getFileSize(pClient->path);
if(size <= pClient->offset){ if (size <= pClient->offset) {
processFileInTheEnd(pClient->pFile, pClient->path); processFileInTheEnd(pClient->pFile, pClient->path);
pClient->pFile = NULL; pClient->pFile = NULL;
}else if(pClient->offset == 0){ } else if (pClient->offset == 0) {
int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL); int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL);
int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL); int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_QUIT, NULL);
tscDebug("[monitor] monitorSendAllSlowLogAtQuit send slow log clusterId:%"PRId64",ret:%d", *clusterId, code); tscDebug("[monitor] monitorSendAllSlowLogAtQuit send slow log clusterId:%" PRId64 ",ret:%d", *clusterId, code);
if (code == 0){ if (code == 0) {
quitCnt ++; quitCnt++;
} }
} }
} }
} }
static void processFileRemoved(SlowLogClient* pClient){ static void processFileRemoved(SlowLogClient* pClient) {
taosUnLockFile(pClient->pFile); taosUnLockFile(pClient->pFile);
taosCloseFile(&(pClient->pFile)); taosCloseFile(&(pClient->pFile));
TdFilePtr pFile = taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC); TdFilePtr pFile =
taosOpenFile(pClient->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_READ | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
tscError("failed to open file:%s since %s", pClient->path, terrstr()); tscError("failed to open file:%s since %s", pClient->path, terrstr());
}else{ } else {
pClient->pFile = pFile; pClient->pFile = pFile;
} }
} }
static void monitorSendAllSlowLog(){ static void monitorSendAllSlowLog() {
int64_t t = taosGetMonoTimestampMs(); int64_t t = taosGetMonoTimestampMs();
void* pIter = NULL; void* pIter = NULL;
while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) { while ((pIter = taosHashIterate(monitorSlowLogHash, pIter))) {
int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL); int64_t* clusterId = (int64_t*)taosHashGetKey(pIter, NULL);
SAppInstInfo* pInst = getAppInstByClusterId(*clusterId); SAppInstInfo* pInst = getAppInstByClusterId(*clusterId);
SlowLogClient* pClient = (*(SlowLogClient**)pIter); SlowLogClient* pClient = (*(SlowLogClient**)pIter);
if (pClient == NULL){ if (pClient == NULL) {
taosHashCancelIterate(monitorSlowLogHash, pIter); taosHashCancelIterate(monitorSlowLogHash, pIter);
return; return;
} }
if (t - pClient->lastCheckTime > pInst->monitorParas.tsMonitorInterval * 1000){ if (t - pClient->lastCheckTime > pInst->monitorParas.tsMonitorInterval * 1000) {
pClient->lastCheckTime = t; pClient->lastCheckTime = t;
} else { } else {
continue; continue;
@ -595,35 +606,35 @@ static void monitorSendAllSlowLog(){
if (pInst != NULL && pClient->offset == 0) { if (pInst != NULL && pClient->offset == 0) {
int64_t size = getFileSize(pClient->path); int64_t size = getFileSize(pClient->path);
if(size <= 0){ if (size <= 0) {
if(size < 0){ if (size < 0) {
tscError("[monitor] monitorSendAllSlowLog failed to get file size:%s, err:%d", pClient->path, errno); tscError("[monitor] monitorSendAllSlowLog failed to get file size:%s, err:%d", pClient->path, errno);
if(errno == ENOENT){ if (errno == ENOENT) {
processFileRemoved(pClient); processFileRemoved(pClient);
} }
} }
continue; continue;
} }
int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL); int32_t code = monitorReadSend(*clusterId, pClient->pFile, &pClient->offset, size, SLOW_LOG_READ_RUNNING, NULL);
tscDebug("[monitor] monitorSendAllSlowLog send slow log clusterId:%"PRId64",ret:%d", *clusterId, code); tscDebug("[monitor] monitorSendAllSlowLog send slow log clusterId:%" PRId64 ",ret:%d", *clusterId, code);
} }
} }
} }
static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId) {
SAppInstInfo* pInst = getAppInstByClusterId((int64_t)clusterId); SAppInstInfo* pInst = getAppInstByClusterId((int64_t)clusterId);
if(pInst == NULL || !pInst->monitorParas.tsEnableMonitor){ if (pInst == NULL || !pInst->monitorParas.tsEnableMonitor) {
tscInfo("[monitor] monitor is disabled, skip send slow log"); tscInfo("[monitor] monitor is disabled, skip send slow log");
return; return;
} }
char namePrefix[PATH_MAX] = {0}; char namePrefix[PATH_MAX] = {0};
if (snprintf(namePrefix, sizeof(namePrefix), "%s%"PRIx64, TD_TMP_FILE_PREFIX, clusterId) < 0) { if (snprintf(namePrefix, sizeof(namePrefix), "%s%" PRIx64, TD_TMP_FILE_PREFIX, clusterId) < 0) {
tscError("failed to generate slow log file name prefix"); tscError("failed to generate slow log file name prefix");
return; return;
} }
char tmpPath[PATH_MAX] = {0}; char tmpPath[PATH_MAX] = {0};
if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0) { if (getSlowLogTmpDir(tmpPath, sizeof(tmpPath)) < 0) {
return; return;
} }
@ -639,11 +650,9 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
continue; continue;
} }
char *name = taosGetDirEntryName(de); char* name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || strstr(name, namePrefix) == NULL) {
strcmp(name, "..") == 0 || tscInfo("skip file:%s, for cluster id:%" PRIx64, name, clusterId);
strstr(name, namePrefix) == NULL) {
tscInfo("skip file:%s, for cluster id:%"PRIx64, name, clusterId);
continue; continue;
} }
@ -659,7 +668,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
taosCloseFile(&pFile); taosCloseFile(&pFile);
continue; continue;
} }
char *tmp = taosStrdup(filename); char* tmp = taosStrdup(filename);
monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0); monitorSendSlowLogAtBeginning(clusterId, &tmp, pFile, 0);
taosMemoryFree(tmp); taosMemoryFree(tmp);
} }
@ -667,7 +676,7 @@ static void monitorSendAllSlowLogFromTempDir(int64_t clusterId){
taosCloseDir(&pDir); taosCloseDir(&pDir);
} }
static void* monitorThreadFunc(void *param){ static void* monitorThreadFunc(void* param) {
setThreadName("client-monitor-slowlog"); setThreadName("client-monitor-slowlog");
#ifdef WINDOWS #ifdef WINDOWS
@ -680,18 +689,18 @@ static void* monitorThreadFunc(void *param){
return NULL; return NULL;
} }
tscDebug("monitorThreadFunc start"); tscDebug("monitorThreadFunc start");
int64_t quitTime = 0; int64_t quitTime = 0;
while (1) { while (1) {
if (atomic_load_32(&slowLogFlag) > 0 > 0) { if (atomic_load_32(&slowLogFlag) > 0 > 0) {
if(quitCnt == 0){ if (quitCnt == 0) {
monitorSendAllSlowLogAtQuit(); monitorSendAllSlowLogAtQuit();
if(quitCnt == 0){ if (quitCnt == 0) {
tscInfo("monitorThreadFunc quit since no slow log to send"); tscInfo("monitorThreadFunc quit since no slow log to send");
break; break;
} }
quitTime = taosGetMonoTimestampMs(); quitTime = taosGetMonoTimestampMs();
} }
if(taosGetMonoTimestampMs() - quitTime > 500){ //quit at most 500ms if (taosGetMonoTimestampMs() - quitTime > 500) { // quit at most 500ms
tscInfo("monitorThreadFunc quit since timeout"); tscInfo("monitorThreadFunc quit since timeout");
break; break;
} }
@ -700,18 +709,19 @@ static void* monitorThreadFunc(void *param){
MonitorSlowLogData* slowLogData = NULL; MonitorSlowLogData* slowLogData = NULL;
taosReadQitem(monitorQueue, (void**)&slowLogData); taosReadQitem(monitorQueue, (void**)&slowLogData);
if (slowLogData != NULL) { if (slowLogData != NULL) {
if (slowLogData->type == SLOW_LOG_READ_BEGINNIG){ if (slowLogData->type == SLOW_LOG_READ_BEGINNIG) {
if(slowLogData->pFile != NULL){ if (slowLogData->pFile != NULL) {
monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile, slowLogData->offset); monitorSendSlowLogAtBeginning(slowLogData->clusterId, &(slowLogData->fileName), slowLogData->pFile,
}else{ slowLogData->offset);
} else {
monitorSendAllSlowLogFromTempDir(slowLogData->clusterId); monitorSendAllSlowLogFromTempDir(slowLogData->clusterId);
} }
} else if(slowLogData->type == SLOW_LOG_WRITE){ } else if (slowLogData->type == SLOW_LOG_WRITE) {
monitorWriteSlowLog2File(slowLogData, tmpSlowLogPath); monitorWriteSlowLog2File(slowLogData, tmpSlowLogPath);
} else if(slowLogData->type == SLOW_LOG_READ_RUNNING){ } else if (slowLogData->type == SLOW_LOG_READ_RUNNING) {
monitorSendSlowLogAtRunning(slowLogData->clusterId); monitorSendSlowLogAtRunning(slowLogData->clusterId);
} else if(slowLogData->type == SLOW_LOG_READ_QUIT){ } else if (slowLogData->type == SLOW_LOG_READ_QUIT) {
if(monitorSendSlowLogAtQuit(slowLogData->clusterId)){ if (monitorSendSlowLogAtQuit(slowLogData->clusterId)) {
tscInfo("monitorThreadFunc quit since all slow log sended"); tscInfo("monitorThreadFunc quit since all slow log sended");
monitorFreeSlowLogData(slowLogData); monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData); taosFreeQitem(slowLogData);
@ -757,8 +767,11 @@ static void tscMonitorStop() {
} }
int32_t monitorInit() { int32_t monitorInit() {
int32_t code;
tscInfo("[monitor] tscMonitor init"); tscInfo("[monitor] tscMonitor init");
monitorCounterHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); monitorCounterHash =
(SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorCounterHash == NULL) { if (monitorCounterHash == NULL) {
tscError("failed to create monitorCounterHash"); tscError("failed to create monitorCounterHash");
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -766,7 +779,8 @@ int32_t monitorInit() {
} }
taosHashSetFreeFp(monitorCounterHash, destroyMonitorClient); taosHashSetFreeFp(monitorCounterHash, destroyMonitorClient);
monitorSlowLogHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); monitorSlowLogHash =
(SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
if (monitorSlowLogHash == NULL) { if (monitorSlowLogHash == NULL) {
tscError("failed to create monitorSlowLogHash"); tscError("failed to create monitorSlowLogHash");
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -781,7 +795,7 @@ int32_t monitorInit() {
return -1; return -1;
} }
if (getSlowLogTmpDir(tmpSlowLogPath, sizeof(tmpSlowLogPath)) < 0){ if (getSlowLogTmpDir(tmpSlowLogPath, sizeof(tmpSlowLogPath)) < 0) {
terrno = TSDB_CODE_TSC_INTERNAL_ERROR; terrno = TSDB_CODE_TSC_INTERNAL_ERROR;
return -1; return -1;
} }
@ -798,14 +812,15 @@ int32_t monitorInit() {
return -1; return -1;
} }
monitorQueue = taosOpenQueue(); code = taosOpenQueue(&monitorQueue);
if(monitorQueue == NULL){ if (code) {
terrno = code;
tscError("open queue error since %s", terrstr()); tscError("open queue error since %s", terrstr());
return -1; return -1;
} }
taosInitRWLatch(&monitorLock); taosInitRWLatch(&monitorLock);
if (tscMonitortInit() != 0){ if (tscMonitortInit() != 0) {
return -1; return -1;
} }
return 0; return 0;
@ -828,21 +843,26 @@ void monitorClose() {
taosWUnLockLatch(&monitorLock); taosWUnLockLatch(&monitorLock);
} }
int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data){ int32_t monitorPutData2MonitorQueue(MonitorSlowLogData data) {
int32_t code;
MonitorSlowLogData* slowLogData;
if (atomic_load_32(&slowLogFlag) == -2) { if (atomic_load_32(&slowLogFlag) == -2) {
tscError("[monitor] slow log thread is exiting"); tscError("[monitor] slow log thread is exiting");
return -1; return -1;
} }
MonitorSlowLogData* slowLogData = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0);
if (slowLogData == NULL) { code = taosAllocateQitem(sizeof(MonitorSlowLogData), DEF_QITEM, 0, (void**)&slowLogData);
if (code) {
tscError("[monitor] failed to allocate slow log data"); tscError("[monitor] failed to allocate slow log data");
return -1; return terrno = code;
} }
*slowLogData = data; *slowLogData = data;
tscDebug("[monitor] write slow log to queue, clusterId:%"PRIx64 " type:%s, data:%s", slowLogData->clusterId, queueTypeStr[slowLogData->type], slowLogData->data); tscDebug("[monitor] write slow log to queue, clusterId:%" PRIx64 " type:%s, data:%s", slowLogData->clusterId,
if (taosWriteQitem(monitorQueue, slowLogData) == 0){ queueTypeStr[slowLogData->type], slowLogData->data);
if (taosWriteQitem(monitorQueue, slowLogData) == 0) {
tsem2_post(&monitorSem); tsem2_post(&monitorSem);
}else{ } else {
monitorFreeSlowLogData(slowLogData); monitorFreeSlowLogData(slowLogData);
taosFreeQitem(slowLogData); taosFreeQitem(slowLogData);
} }

35
source/client/src/clientRawBlockWrite.c
View File

@ -1116,9 +1116,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taosHashGetSize(pVgroupHashmap) == 0) { if (taosHashGetSize(pVgroupHashmap) == 0) {
goto end; goto end;
} }
SArray* pBufArray = serializeVgroupsCreateTableBatch(pVgroupHashmap); SArray* pBufArray = NULL;
if (NULL == pBufArray) { code = serializeVgroupsCreateTableBatch(pVgroupHashmap, &pBufArray);
code = TSDB_CODE_OUT_OF_MEMORY; if (TSDB_CODE_SUCCESS != code) {
goto end; goto end;
} }
@ -1264,9 +1264,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) {
if (taosHashGetSize(pVgroupHashmap) == 0) { if (taosHashGetSize(pVgroupHashmap) == 0) {
goto end; goto end;
} }
SArray* pBufArray = serializeVgroupsDropTableBatch(pVgroupHashmap); SArray* pBufArray = NULL;
if (NULL == pBufArray) { code = serializeVgroupsDropTableBatch(pVgroupHashmap, &pBufArray);
code = TSDB_CODE_OUT_OF_MEMORY; if (TSDB_CODE_SUCCESS != code) {
goto end; goto end;
} }
@ -1572,9 +1572,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pDat
} }
// uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid); // uError("td23101 0vgId:%d, vgId:%d, name:%s, uid:%"PRIu64, vgData.vgId, pTableMeta->vgId, tbname, pTableMeta->uid);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1656,9 +1655,9 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
pQuery = smlInitHandle();
if (pQuery == NULL) { code = smlInitHandle(&pQuery);
code = TSDB_CODE_OUT_OF_MEMORY; if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
@ -1755,12 +1754,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
conn.requestObjRefId = pRequest->self; conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_OUT_OF_MEMORY;
SET_ERROR_MSG("init sml handle failed"); SET_ERROR_MSG("init sml handle failed");
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) { while (++rspObj.common.resIter < rspObj.rsp.common.blockNum) {
void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter); void* pRetrieve = taosArrayGetP(rspObj.rsp.common.blockData, rspObj.common.resIter);
@ -1903,12 +1902,12 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
conn.requestObjRefId = pRequest->self; conn.requestObjRefId = pRequest->self;
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); code = smlInitHandle(&pQuery);
if (pQuery == NULL) { if (code != TSDB_CODE_SUCCESS) {
SET_ERROR_MSG("init sml handle failed"); SET_ERROR_MSG("init sml handle failed");
code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum); uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.common.blockNum);

637
source/client/src/clientSml.c
View File

File diff suppressed because it is too large Load Diff

118
source/client/src/clientSmlJson.c
View File

@ -29,29 +29,29 @@
(start)++; \ (start)++; \
} }
static char *smlJsonGetObj(char *payload) { static int32_t smlJsonGetObj(char **payload) {
int leftBracketCnt = 0; int leftBracketCnt = 0;
bool isInQuote = false; bool isInQuote = false;
while (*payload) { while (**payload) {
if (*payload == '"' && *(payload - 1) != '\\') { if (**payload == '"' && *((*payload) - 1) != '\\') {
isInQuote = !isInQuote; isInQuote = !isInQuote;
} else if (!isInQuote && unlikely(*payload == '{')) { } else if (!isInQuote && unlikely(**payload == '{')) {
leftBracketCnt++; leftBracketCnt++;
payload++; (*payload)++;
continue; continue;
} else if (!isInQuote && unlikely(*payload == '}')) { } else if (!isInQuote && unlikely(**payload == '}')) {
leftBracketCnt--; leftBracketCnt--;
payload++; (*payload)++;
if (leftBracketCnt == 0) { if (leftBracketCnt == 0) {
return payload; return 0;
} else if (leftBracketCnt < 0) { } else if (leftBracketCnt < 0) {
return NULL; return -1;
} }
continue; continue;
} }
payload++; (*payload)++;
} }
return NULL; return -1;
} }
int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
@ -99,8 +99,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->timestamp = (*start); element->timestamp = (*start);
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start); element->timestampLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -127,8 +128,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->cols = (*start); element->cols = (*start);
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start); element->colsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -153,8 +155,9 @@ int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) {
JUMP_JSON_SPACE((*start)) JUMP_JSON_SPACE((*start))
offset[index++] = *start - sTmp; offset[index++] = *start - sTmp;
element->tags = (*start); element->tags = (*start);
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start); element->tagsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -209,8 +212,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++]; (*start) += offset[index++];
element->timestamp = *start; element->timestamp = *start;
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->timestampLen = tmp - (*start); element->timestampLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -227,8 +231,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
(*start) += offset[index++]; (*start) += offset[index++];
element->cols = *start; element->cols = *start;
if (*(*start) == '{') { if (*(*start) == '{') {
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->colsLen = tmp - (*start); element->colsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -244,8 +249,9 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
} else if ((*start)[1] == 't' && (*start)[2] == 'a') { } else if ((*start)[1] == 't' && (*start)[2] == 'a') {
(*start) += offset[index++]; (*start) += offset[index++];
element->tags = (*start); element->tags = (*start);
char *tmp = smlJsonGetObj((*start)); char *tmp = *start;
if (tmp) { int32_t code = smlJsonGetObj(&tmp);
if (code == 0) {
element->tagsLen = tmp - (*start); element->tagsLen = tmp - (*start);
*start = tmp; *start = tmp;
} }
@ -261,7 +267,7 @@ int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) {
uError("elements != %d", OTD_JSON_FIELDS_NUM); uError("elements != %d", OTD_JSON_FIELDS_NUM);
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) {
@ -466,7 +472,11 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
break; break;
} }
case cJSON_String: { case cJSON_String: {
smlConvertJSONString(kv, "binary", root); int32_t ret = smlConvertJSONString(kv, "binary", root);
if (ret != TSDB_CODE_SUCCESS) {
uError("OTD:Failed to parse binary value from JSON Obj");
return ret;
}
break; break;
} }
case cJSON_Object: { case cJSON_Object: {
@ -492,20 +502,17 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
int32_t tagNum = cJSON_GetArraySize(tags); int32_t tagNum = cJSON_GetArraySize(tags);
if (unlikely(tagNum == 0)) { if (unlikely(tagNum == 0)) {
uError("SML:Tag should not be empty"); uError("SML:Tag should not be empty");
terrno = TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
for (int32_t i = 0; i < tagNum; ++i) { for (int32_t i = 0; i < tagNum; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i); cJSON *tag = cJSON_GetArrayItem(tags, i);
if (unlikely(tag == NULL)) { if (unlikely(tag == NULL)) {
terrno = TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
size_t keyLen = strlen(tag->string); size_t keyLen = strlen(tag->string);
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
uError("OTD:Tag key length is 0 or too large than 64"); uError("OTD:Tag key length is 0 or too large than 64");
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// add kv to SSmlKv // add kv to SSmlKv
@ -516,19 +523,19 @@ static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
// value // value
int32_t ret = smlParseValueFromJSON(tag, &kv); int32_t ret = smlParseValueFromJSON(tag, &kv);
if (unlikely(ret != TSDB_CODE_SUCCESS)) { if (unlikely(ret != TSDB_CODE_SUCCESS)) {
terrno = ret; return ret;
return -1; }
if (taosArrayPush(preLineKV, &kv) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(preLineKV, &kv);
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
cnt++; cnt++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
@ -536,12 +543,18 @@ static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagJson(info, tags); ret = smlProcessTagJson(info, tags);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
ret = smlJoinMeasureTag(elements); ret = smlJoinMeasureTag(elements);
if(ret != 0){ if(ret != 0){
@ -703,6 +716,9 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// Parse tags // Parse tags
bool needFree = info->dataFormat; bool needFree = info->dataFormat;
elements->tags = cJSON_PrintUnformatted(tagsJson); elements->tags = cJSON_PrintUnformatted(tagsJson);
if (elements->tags == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
elements->tagsLen = strlen(elements->tags); elements->tagsLen = strlen(elements->tags);
if (is_same_child_table_telnet(elements, &info->preLine) != 0) { if (is_same_child_table_telnet(elements, &info->preLine) != 0) {
ret = smlParseTagsFromJSON(info, tagsJson, elements); ret = smlParseTagsFromJSON(info, tagsJson, elements);
@ -729,7 +745,8 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
// notice!!! put ts back to tag to ensure get meta->precision // notice!!! put ts back to tag to ensure get meta->precision
int64_t ts = smlParseTSFromJSON(info, tsJson); int64_t ts = smlParseTSFromJSON(info, tsJson);
if (unlikely(ts < 0)) { if (unlikely(ts < 0)) {
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); char* tmp = cJSON_PrintUnformatted(tsJson);
uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload %s %s %" PRId64, info->id, info->msgBuf.buf,tmp, ts);
return TSDB_CODE_INVALID_TIMESTAMP; return TSDB_CODE_INVALID_TIMESTAMP;
} }
SSmlKv kvTs = {0}; SSmlKv kvTs = {0};
@ -823,7 +840,7 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
} }
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_VALUE; return ret;
} }
if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS;
@ -844,18 +861,23 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
cJSON *valueJson = cJSON_Parse(elements->cols); cJSON *valueJson = cJSON_Parse(elements->cols);
if (unlikely(valueJson == NULL)) { if (unlikely(valueJson == NULL)) {
uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
taosArrayPush(info->tagJsonArray, &valueJson); if (taosArrayPush(info->tagJsonArray, &valueJson) == NULL){
cJSON_Delete(valueJson);
elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseValueFromJSONObj(valueJson, &kv); ret = smlParseValueFromJSONObj(valueJson, &kv);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); uError("SML:0x%" PRIx64 " Failed to parse value from JSON Obj:%s", info->id, elements->cols);
elements->cols[elements->colsLen] = tmp; elements->cols[elements->colsLen] = tmp;
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
elements->cols[elements->colsLen] = tmp; elements->cols[elements->colsLen] = tmp;
} else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) {
uError("SML:cols invalidate:%s", elements->cols); uError("SML:0x%" PRIx64 " cols invalidate:%s", info->id, elements->cols);
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
@ -870,7 +892,11 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
return TSDB_CODE_TSC_INVALID_JSON; return TSDB_CODE_TSC_INVALID_JSON;
} }
taosArrayPush(info->tagJsonArray, &tagsJson); if (taosArrayPush(info->tagJsonArray, &tagsJson) == NULL){
cJSON_Delete(tagsJson);
uError("SML:0x%" PRIx64 " taosArrayPush failed", info->id);
return TSDB_CODE_OUT_OF_MEMORY;
}
ret = smlParseTagsFromJSON(info, tagsJson, elements); ret = smlParseTagsFromJSON(info, tagsJson, elements);
if (unlikely(ret)) { if (unlikely(ret)) {
uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id);
@ -937,7 +963,7 @@ int32_t smlParseJSON(SSmlHandle *info, char *payload) {
return ret; return ret;
} }
info->lines = (SSmlLineInfo *)tmp; info->lines = (SSmlLineInfo *)tmp;
memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); (void)memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo));
} }
ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt);
if ((info->lines + cnt)->measure == NULL) break; if ((info->lines + cnt)->measure == NULL) break;

68
source/client/src/clientSmlLine.c
View File

@ -103,7 +103,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
return code; return code;
} }
char* tmp = taosMemoryCalloc(pVal->length, 1); char* tmp = taosMemoryCalloc(pVal->length, 1);
memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, pVal->value + NCHAR_ADD_LEN - 1, pVal->length - NCHAR_ADD_LEN);
code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length); code = doGeomFromText(tmp, (unsigned char **)&pVal->value, &pVal->length);
taosMemoryFree(tmp); taosMemoryFree(tmp);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -149,7 +152,7 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
if(data == NULL){ if(data == NULL){
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length); (void)memcpy(data, pVal->value + (NCHAR_ADD_LEN - 1), pVal->length);
pVal->value = data; pVal->value = data;
} }
@ -212,8 +215,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
while (*sql < sqlEnd) { while (*sql < sqlEnd) {
if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) { if (unlikely(IS_SPACE(*sql,escapeChar) || IS_COMMA(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(IS_EQUAL(*sql,escapeChar))) { if (unlikely(IS_EQUAL(*sql,escapeChar))) {
keyLen = *sql - key; keyLen = *sql - key;
@ -230,8 +232,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) { if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// parse value // parse value
@ -245,8 +246,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
break; break;
} else if (unlikely(IS_EQUAL(*sql,escapeChar))) { } else if (unlikely(IS_EQUAL(*sql,escapeChar))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) { if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@ -261,24 +261,28 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
if (unlikely(valueLen == 0)) { if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
} }
if (keyEscaped) { if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(keyLen); char *tmp = (char *)taosMemoryMalloc(keyLen);
memcpy(tmp, key, keyLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, keyLen);
key = tmp; key = tmp;
} }
if (valueEscaped) { if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(valueLen); char *tmp = (char *)taosMemoryMalloc(valueLen);
memcpy(tmp, value, valueLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, value, valueLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, valueLen);
value = tmp; value = tmp;
} }
@ -289,11 +293,12 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
.length = valueLen, .length = valueLen,
.keyEscaped = keyEscaped, .keyEscaped = keyEscaped,
.valueEscaped = valueEscaped}; .valueEscaped = valueEscaped};
taosArrayPush(preLineKV, &kv); if(taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_TSC_INVALID_JSON;
return -1;
} }
cnt++; cnt++;
@ -302,7 +307,7 @@ static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
} }
(*sql)++; (*sql)++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) { static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
@ -315,13 +320,19 @@ static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagLine(info, sql, sqlEnd); ret = smlProcessTagLine(info, sql, sqlEnd);
if(ret != 0){ if(ret != 0){
return terrno; if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
return smlProcessChildTable(info, elements); return smlProcessChildTable(info, elements);
@ -410,7 +421,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (keyEscaped) { if (keyEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.keyLen); char *tmp = (char *)taosMemoryMalloc(kv.keyLen);
memcpy(tmp, key, kv.keyLen); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, key, kv.keyLen);
PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen); PROCESS_SLASH_IN_TAG_FIELD_KEY(tmp, kv.keyLen);
kv.key = tmp; kv.key = tmp;
kv.keyEscaped = keyEscaped; kv.keyEscaped = keyEscaped;
@ -418,7 +432,10 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
if (valueEscaped) { if (valueEscaped) {
char *tmp = (char *)taosMemoryMalloc(kv.length); char *tmp = (char *)taosMemoryMalloc(kv.length);
memcpy(tmp, kv.value, kv.length); if (tmp == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
(void)memcpy(tmp, kv.value, kv.length);
PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length); PROCESS_SLASH_IN_FIELD_VALUE(tmp, kv.length);
ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY); ASSERT(kv.type != TSDB_DATA_TYPE_GEOMETRY);
if(kv.type == TSDB_DATA_TYPE_VARBINARY){ if(kv.type == TSDB_DATA_TYPE_VARBINARY){
@ -437,8 +454,13 @@ static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlL
} else { } else {
if (currElement->colArray == NULL) { if (currElement->colArray == NULL) {
currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
if (currElement->colArray == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (taosArrayPush(currElement->colArray, &kv) == NULL){ // reserve for timestamp
return TSDB_CODE_OUT_OF_MEMORY;
} }
taosArrayPush(currElement->colArray, &kv); // reserve for timestamp
} }
cnt++; cnt++;

34
source/client/src/clientSmlTelnet.c
View File

@ -84,8 +84,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
while (sql < sqlEnd) { while (sql < sqlEnd) {
if (unlikely(*sql == SPACE)) { if (unlikely(*sql == SPACE)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
if (unlikely(*sql == EQUAL)) { if (unlikely(*sql == EQUAL)) {
keyLen = sql - key; keyLen = sql - key;
@ -97,8 +96,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(IS_INVALID_COL_LEN(keyLen))) { if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
return -1;
} }
// parse value // parse value
@ -111,8 +109,7 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
} }
if (unlikely(*sql == EQUAL)) { if (unlikely(*sql == EQUAL)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
terrno = TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
sql++; sql++;
} }
@ -120,13 +117,11 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
if (unlikely(valueLen == 0)) { if (unlikely(valueLen == 0)) {
smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
terrno = TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
return -1;
} }
if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
return -1;
} }
SSmlKv kv = {.key = key, SSmlKv kv = {.key = key,
@ -136,14 +131,15 @@ static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
.length = valueLen, .length = valueLen,
.keyEscaped = false, .keyEscaped = false,
.valueEscaped = false}; .valueEscaped = false};
taosArrayPush(preLineKV, &kv); if (taosArrayPush(preLineKV, &kv) == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) { if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
terrno = TSDB_CODE_SUCCESS; return TSDB_CODE_SML_INVALID_DATA;
return -1;
} }
cnt++; cnt++;
} }
return 0; return TSDB_CODE_SUCCESS;
} }
static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) { static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
@ -156,13 +152,19 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
if(info->dataFormat){ if(info->dataFormat){
ret = smlProcessSuperTable(info, elements); ret = smlProcessSuperTable(info, elements);
if(ret != 0){ if(ret != 0){
return terrno; if(info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
} }
ret = smlProcessTagTelnet(info, data, sqlEnd); ret = smlProcessTagTelnet(info, data, sqlEnd);
if(ret != 0){ if(ret != 0){
return terrno; if (info->reRun){
return TSDB_CODE_SUCCESS;
}
return ret;
} }
ret = smlJoinMeasureTag(elements); ret = smlJoinMeasureTag(elements);

102
source/client/src/clientTmq.c
View File

@ -733,13 +733,17 @@ end:
} }
static void generateTimedTask(int64_t refId, int32_t type) { static void generateTimedTask(int64_t refId, int32_t type) {
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); tmq_t* tmq;
int8_t* pTaskType;
int32_t code;
tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) return; if (tmq == NULL) return;
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); code = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0, (void**)&pTaskType);
if (pTaskType != NULL){ if (code == TSDB_CODE_SUCCESS) {
*pTaskType = type; *pTaskType = type;
if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0){ if (taosWriteQitem(tmq->delayedTask, pTaskType) == 0) {
tsem2_post(&tmq->rspSem); tsem2_post(&tmq->rspSem);
} }
} }
@ -877,7 +881,7 @@ void tmqSendHbReq(void* param, void* tmrId) {
OVER: OVER:
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
if(tmrId != NULL){ if (tmrId != NULL) {
taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer); taosTmrReset(tmqSendHbReq, DEFAULT_HEARTBEAT_INTERVAL, param, tmqMgmt.timer, &tmq->hbLiveTimer);
} }
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
@ -890,7 +894,12 @@ static void defaultCommitCbFn(tmq_t* pTmq, int32_t code, void* param) {
} }
int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
STaosQall* qall = taosAllocateQall(); STaosQall* qall;
int32_t code;
code = taosAllocateQall(&qall);
if (code) return code;
taosReadAllQitems(pTmq->delayedTask, qall); taosReadAllQitems(pTmq->delayedTask, qall);
int32_t numOfItems = taosQallItemSize(qall); int32_t numOfItems = taosQallItemSize(qall);
@ -914,7 +923,8 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) {
asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam); asyncCommitAllOffsets(pTmq, pCallbackFn, pTmq->commitCbUserParam);
tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId, tscDebug("consumer:0x%" PRIx64 " next commit to vnode(s) in %.2fs", pTmq->consumerId,
pTmq->autoCommitInterval / 1000.0); pTmq->autoCommitInterval / 1000.0);
taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer, &pTmq->commitTimer); taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, (void*)(pTmq->refId), tmqMgmt.timer,
&pTmq->commitTimer);
} else { } else {
tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType); tscError("consumer:0x%" PRIx64 " invalid task type:%d", pTmq->consumerId, *pTaskType);
} }
@ -977,14 +987,14 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
} }
int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) { if (param == NULL) {
return code; return code;
} }
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code; pParam->rspErr = code;
if(pMsg){ if (pMsg) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
} }
tsem_post(&pParam->rspSem); tsem_post(&pParam->rspSem);
@ -1057,13 +1067,13 @@ void tmqFreeImpl(void* handle) {
taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
taos_close_internal(tmq->pTscObj); taos_close_internal(tmq->pTscObj);
if(tmq->commitTimer) { if (tmq->commitTimer) {
taosTmrStopA(&tmq->commitTimer); taosTmrStopA(&tmq->commitTimer);
} }
if(tmq->epTimer) { if (tmq->epTimer) {
taosTmrStopA(&tmq->epTimer); taosTmrStopA(&tmq->epTimer);
} }
if(tmq->hbLiveTimer) { if (tmq->hbLiveTimer) {
taosTmrStopA(&tmq->hbLiveTimer); taosTmrStopA(&tmq->hbLiveTimer);
} }
taosMemoryFree(tmq); taosMemoryFree(tmq);
@ -1101,6 +1111,8 @@ void tmqMgmtClose(void) {
if (errstr != NULL) snprintf(errstr, errstrLen, MSG); if (errstr != NULL) snprintf(errstr, errstrLen, MSG);
tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
int32_t code;
if (conf == NULL) { if (conf == NULL) {
SET_ERROR_MSG_TMQ("configure is null") SET_ERROR_MSG_TMQ("configure is null")
return NULL; return NULL;
@ -1124,12 +1136,31 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass;
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
pTmq->mqueue = taosOpenQueue(); code = taosOpenQueue(&pTmq->mqueue);
pTmq->delayedTask = taosOpenQueue(); if (code) {
pTmq->qall = taosAllocateQall(); terrno = code;
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
SET_ERROR_MSG_TMQ("open queue failed")
goto _failed;
}
if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || code = taosOpenQueue(&pTmq->delayedTask);
conf->groupId[0] == 0) { if (code) {
terrno = code;
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
SET_ERROR_MSG_TMQ("open delayed task queue failed")
goto _failed;
}
code = taosAllocateQall(&pTmq->qall);
if (code) {
terrno = code;
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
SET_ERROR_MSG_TMQ("allocate qall failed")
goto _failed;
}
if (conf->groupId[0] == 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId); tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty") SET_ERROR_MSG_TMQ("malloc tmq element failed or group is empty")
@ -1315,7 +1346,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
// init ep timer // init ep timer
tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer); tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, DEFAULT_ASKEP_INTERVAL, (void*)(tmq->refId), tmqMgmt.timer);
// init auto commit timer // init auto commit timer
tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer); tmq->commitTimer =
taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, (void*)(tmq->refId), tmqMgmt.timer);
FAIL: FAIL:
taosArrayDestroyP(req.topicNames, taosMemoryFree); taosArrayDestroyP(req.topicNames, taosMemoryFree);
@ -1372,20 +1404,20 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (pParam == NULL || pMsg == NULL) { if (pParam == NULL || pMsg == NULL) {
goto FAIL; goto FAIL;
} }
int64_t refId = pParam->refId; int64_t refId = pParam->refId;
int32_t vgId = pParam->vgId; int32_t vgId = pParam->vgId;
uint64_t requestId = pParam->requestId; uint64_t requestId = pParam->requestId;
tmq = taosAcquireRef(tmqMgmt.rsetId, refId); tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
if (tmq == NULL) { if (tmq == NULL) {
code = TSDB_CODE_TMQ_CONSUMER_CLOSED; code = TSDB_CODE_TMQ_CONSUMER_CLOSED;
goto FAIL; goto FAIL;
} }
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0); SMqPollRspWrapper* pRspWrapper;
if (pRspWrapper == NULL) { code = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0, (void**)&pRspWrapper);
if (code) {
tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId); tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d, since out of memory", tmq->consumerId, vgId);
taosReleaseRef(tmqMgmt.rsetId, refId); taosReleaseRef(tmqMgmt.rsetId, refId);
code = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL; goto FAIL;
} }
@ -2575,7 +2607,7 @@ end:
} }
int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) return code; if (param == NULL) return code;
SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param;
tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId); tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, pParam->refId);
if (tmq == NULL) { if (tmq == NULL) {
@ -2600,9 +2632,9 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) {
doUpdateLocalEp(tmq, head->epoch, &rsp); doUpdateLocalEp(tmq, head->epoch, &rsp);
tDeleteSMqAskEpRsp(&rsp); tDeleteSMqAskEpRsp(&rsp);
} else { } else {
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); SMqAskEpRspWrapper* pWrapper;
if (pWrapper == NULL) { code = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0, (void**)&pWrapper);
code = TSDB_CODE_OUT_OF_MEMORY; if (code) {
goto END; goto END;
} }
@ -2620,13 +2652,13 @@ END:
FAIL: FAIL:
if (pParam->sync) { if (pParam->sync) {
SAskEpInfo* pInfo = pParam->pParam; SAskEpInfo* pInfo = pParam->pParam;
if(pInfo) { if (pInfo) {
pInfo->code = code; pInfo->code = code;
tsem_post(&pInfo->sem); tsem_post(&pInfo->sem);
} }
} }
if(pMsg){ if (pMsg) {
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
} }
@ -2636,11 +2668,11 @@ FAIL:
int32_t syncAskEp(tmq_t* pTmq) { int32_t syncAskEp(tmq_t* pTmq) {
SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo)); SAskEpInfo* pInfo = taosMemoryMalloc(sizeof(SAskEpInfo));
if(pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY; if (pInfo == NULL) return TSDB_CODE_OUT_OF_MEMORY;
tsem_init(&pInfo->sem, 0, 0); tsem_init(&pInfo->sem, 0, 0);
int32_t code = askEp(pTmq, pInfo, true, false); int32_t code = askEp(pTmq, pInfo, true, false);
if(code == 0){ if (code == 0) {
tsem_wait(&pInfo->sem); tsem_wait(&pInfo->sem);
code = pInfo->code; code = pInfo->code;
} }
@ -2778,7 +2810,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
} }
static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
if(param == NULL) { if (param == NULL) {
return code; return code;
} }
SMqVgWalInfoParam* pParam = param; SMqVgWalInfoParam* pParam = param;
@ -2812,7 +2844,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) {
tsem_post(&pCommon->rsp); tsem_post(&pCommon->rsp);
} }
if(pMsg){ if (pMsg) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
} }
@ -3248,7 +3280,7 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);
} }
if(param == NULL) { if (param == NULL) {
return code; return code;
} }
SMqSeekParam* pParam = param; SMqSeekParam* pParam = param;

114
source/client/test/smlTest.cpp
View File

@ -40,13 +40,16 @@ TEST(testCase, smlParseInfluxString_Test) {
msgBuf.len = 256; msgBuf.len = 256;
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, 0);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
// case 1 // case 1
char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3";
char *sql = (char *)taosMemoryCalloc(256, 1); char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, tmp, strlen(tmp) + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, tmp, strlen(tmp) + 1);
int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
@ -62,22 +65,22 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 2 false // case 2 false
tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 4 tag is null // case 4 tag is null
tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measure, sql);
@ -92,13 +95,13 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 5 tag is null // case 5 tag is null
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measure, sql + 1);
@ -112,36 +115,36 @@ TEST(testCase, smlParseInfluxString_Test) {
ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2);
ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000"));
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 6 // case 6
tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
smlClearForRerun(info); ASSERT_EQ(smlClearForRerun(info), 0);
// case 7 // case 7
tmp = " st , "; tmp = " st , ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
// case 8 false // case 8 false
tmp = ", st , "; tmp = ", st , ";
memcpy(sql, tmp, strlen(tmp) + 1); (void)memcpy(sql, tmp, strlen(tmp) + 1);
memset(&elements, 0, sizeof(SSmlLineInfo)); (void)memset(&elements, 0, sizeof(SSmlLineInfo));
ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements);
ASSERT_NE(ret, 0); ASSERT_NE(ret, 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
elements.colArray = NULL; elements.colArray = nullptr;
taosMemoryFree(sql); taosMemoryFree(sql);
smlDestroyInfo(info); smlDestroyInfo(info);
@ -228,7 +231,9 @@ TEST(testCase, smlParseCols_Error_Test) {
"escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000" "escape_test,tag1=\"tag1_value\",tag2=\"tag2_value\" col0=\"co\"l\"0_value\",col1=\"col1_value\" 1680918783010000000"
}; };
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) {
@ -238,13 +243,14 @@ TEST(testCase, smlParseCols_Error_Test) {
msgBuf.len = 256; msgBuf.len = 256;
int32_t len = strlen(data[i]); int32_t len = strlen(data[i]);
char *sql = (char *)taosMemoryCalloc(256, 1); char *sql = (char *)taosMemoryCalloc(256, 1);
memcpy(sql, data[i], len + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data[i], len + 1);
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
// printf("i:%d\n", i); // printf("i:%d\n", i);
ASSERT_NE(ret, TSDB_CODE_SUCCESS); ASSERT_NE(ret, TSDB_CODE_SUCCESS);
taosMemoryFree(sql); taosMemoryFree(sql);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
} }
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -254,7 +260,9 @@ TEST(testCase, smlParseCols_Test) {
SSmlMsgBuf msgBuf; SSmlMsgBuf msgBuf;
msgBuf.buf = msg; msgBuf.buf = msg;
msgBuf.len = 256; msgBuf.len = 256;
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->protocol = TSDB_SML_LINE_PROTOCOL; info->protocol = TSDB_SML_LINE_PROTOCOL;
info->dataFormat = false; info->dataFormat = false;
SSmlLineInfo elements = {0}; SSmlLineInfo elements = {0};
@ -267,7 +275,8 @@ TEST(testCase, smlParseCols_Test) {
"cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000";
int32_t len = strlen(data); int32_t len = strlen(data);
char *sql = (char *)taosMemoryCalloc(1024, 1); char *sql = (char *)taosMemoryCalloc(1024, 1);
memcpy(sql, data, len + 1); ASSERT_NE(sql, nullptr);
(void)memcpy(sql, data, len + 1);
int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements);
ASSERT_EQ(ret, TSDB_CODE_SUCCESS); ASSERT_EQ(ret, TSDB_CODE_SUCCESS);
int32_t size = taosArrayGetSize(elements.colArray); int32_t size = taosArrayGetSize(elements.colArray);
@ -275,6 +284,7 @@ TEST(testCase, smlParseCols_Test) {
// binary // binary
SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY);
@ -285,6 +295,7 @@ TEST(testCase, smlParseCols_Test) {
// nchar // nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); kv = (SSmlKv *)taosArrayGet(elements.colArray, 2);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
@ -293,6 +304,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); kv = (SSmlKv *)taosArrayGet(elements.colArray, 3);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -301,6 +313,7 @@ TEST(testCase, smlParseCols_Test) {
// double // double
kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); kv = (SSmlKv *)taosArrayGet(elements.colArray, 4);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -310,6 +323,7 @@ TEST(testCase, smlParseCols_Test) {
// float // float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); kv = (SSmlKv *)taosArrayGet(elements.colArray, 5);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE);
@ -319,6 +333,7 @@ TEST(testCase, smlParseCols_Test) {
// float // float
kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); kv = (SSmlKv *)taosArrayGet(elements.colArray, 6);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT);
@ -328,6 +343,7 @@ TEST(testCase, smlParseCols_Test) {
// tiny int // tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); kv = (SSmlKv *)taosArrayGet(elements.colArray, 7);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0);
ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT);
@ -336,6 +352,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned tiny int // unsigned tiny int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); kv = (SSmlKv *)taosArrayGet(elements.colArray, 8);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0);
ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->keyLen, 3);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT);
@ -344,6 +361,7 @@ TEST(testCase, smlParseCols_Test) {
// small int // small int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); kv = (SSmlKv *)taosArrayGet(elements.colArray, 9);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT);
@ -352,6 +370,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned smallint // unsigned smallint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); kv = (SSmlKv *)taosArrayGet(elements.colArray, 10);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT);
@ -360,6 +379,7 @@ TEST(testCase, smlParseCols_Test) {
// int // int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); kv = (SSmlKv *)taosArrayGet(elements.colArray, 11);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT);
@ -368,6 +388,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned int // unsigned int
kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); kv = (SSmlKv *)taosArrayGet(elements.colArray, 12);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT);
@ -376,6 +397,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint // bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); kv = (SSmlKv *)taosArrayGet(elements.colArray, 13);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -384,6 +406,7 @@ TEST(testCase, smlParseCols_Test) {
// bigint // bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); kv = (SSmlKv *)taosArrayGet(elements.colArray, 14);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0);
ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->keyLen, 2);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT);
@ -392,6 +415,7 @@ TEST(testCase, smlParseCols_Test) {
// unsigned bigint // unsigned bigint
kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); kv = (SSmlKv *)taosArrayGet(elements.colArray, 15);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0);
ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->keyLen, 4);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT);
@ -400,6 +424,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); kv = (SSmlKv *)taosArrayGet(elements.colArray, 16);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0);
ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->keyLen, 9);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -408,6 +433,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); kv = (SSmlKv *)taosArrayGet(elements.colArray, 17);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0);
ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -416,6 +442,7 @@ TEST(testCase, smlParseCols_Test) {
// bool // bool
kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); kv = (SSmlKv *)taosArrayGet(elements.colArray, 18);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0);
ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->keyLen, 6);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL);
@ -424,13 +451,14 @@ TEST(testCase, smlParseCols_Test) {
// nchar // nchar
kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); kv = (SSmlKv *)taosArrayGet(elements.colArray, 19);
ASSERT_NE(kv, nullptr);
ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0);
ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->keyLen, 5);
ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR);
ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->length, 4);
ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0);
taosArrayDestroy(elements.colArray); (void)taosArrayDestroy(elements.colArray);
taosMemoryFree(sql); taosMemoryFree(sql);
smlDestroyInfo(info); smlDestroyInfo(info);
} }
@ -471,7 +499,9 @@ TEST(testCase, smlParseNumber_Test) {
} }
TEST(testCase, smlParseTelnetLine_error_Test) { TEST(testCase, smlParseTelnetLine_error_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false; info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL; info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
@ -509,7 +539,9 @@ TEST(testCase, smlParseTelnetLine_error_Test) {
} }
TEST(testCase, smlParseTelnetLine_Test) { TEST(testCase, smlParseTelnetLine_Test) {
SSmlHandle *info = smlBuildSmlInfo(NULL); SSmlHandle *info = nullptr;
int32_t code = smlBuildSmlInfo(nullptr, &info);
ASSERT_EQ(code, TSDB_CODE_SUCCESS);
info->dataFormat = false; info->dataFormat = false;
info->protocol = TSDB_SML_TELNET_PROTOCOL; info->protocol = TSDB_SML_TELNET_PROTOCOL;
ASSERT_NE(info, nullptr); ASSERT_NE(info, nullptr);
@ -528,7 +560,9 @@ TEST(testCase, smlParseTelnetLine_Test) {
} }
//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { //TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) {
// SSmlHandle *info = smlBuildSmlInfo(NULL); // SSmlHandle *info = nullptr;
// int32_t code = smlBuildSmlInfo(nullptr, &info);
// ASSERT_EQ(code, TSDB_CODE_SUCCESS);
// info->protocol = TSDB_SML_JSON_PROTOCOL; // info->protocol = TSDB_SML_JSON_PROTOCOL;
// ASSERT_NE(info, nullptr); // ASSERT_NE(info, nullptr);
// //
@ -568,7 +602,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) { for (int j = 0; j < 10000000; ++j) {
kv.value = str[i]; kv.value = str[i];
kv.length = strlen(str[i]); kv.length = strlen(str[i]);
smlParseNumber(&kv, &msgBuf); (void)smlParseNumber(&kv, &msgBuf); //ignore result
} }
printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1);
printf("\n"); printf("\n");
@ -576,7 +610,7 @@ TEST(testCase, smlParseNumber_performance_Test) {
for (int j = 0; j < 10000000; ++j) { for (int j = 0; j < 10000000; ++j) {
kv.value = str[i]; kv.value = str[i];
kv.length = strlen(str[i]); kv.length = strlen(str[i]);
smlParseNumberOld(&kv, &msgBuf); (void)smlParseNumberOld(&kv, &msgBuf); //ignore result
} }
printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2);
printf("\n\n"); printf("\n\n");

183
source/common/src/cos.c
View File

@ -3,6 +3,7 @@
#include "cos.h" #include "cos.h"
#include "cos_cp.h" #include "cos_cp.h"
#include "tdef.h" #include "tdef.h"
#include "tutil.h"
extern char tsS3Endpoint[]; extern char tsS3Endpoint[];
extern char tsS3AccessKeyId[]; extern char tsS3AccessKeyId[];
@ -40,7 +41,7 @@ int32_t s3Begin() {
if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) {
uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); uError("Failed to initialize libs3: %s\n", S3_get_status_name(status));
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
protocolG = !tsS3Https; protocolG = !tsS3Https;
@ -48,12 +49,12 @@ int32_t s3Begin() {
uriStyleG = S3UriStyleVirtualHost; uriStyleG = S3UriStyleVirtualHost;
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void s3End() { S3_deinitialize(); } void s3End() { S3_deinitialize(); }
int32_t s3Init() { return 0; /*s3Begin();*/ } int32_t s3Init() { TAOS_RETURN(TSDB_CODE_SUCCESS); /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/ void s3CleanUp() { /*s3End();*/
} }
@ -61,7 +62,7 @@ void s3CleanUp() { /*s3End();*/
static int32_t s3ListBucket(char const *bucketname); static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() { int32_t s3CheckCfg() {
int32_t code = 0; int32_t code = 0, lino = 0;
if (!tsS3Enabled) { if (!tsS3Enabled) {
fprintf(stderr, "s3 not configured.\n"); fprintf(stderr, "s3 not configured.\n");
@ -71,7 +72,7 @@ int32_t s3CheckCfg() {
code = s3Begin(); code = s3Begin();
if (code != 0) { if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n"); fprintf(stderr, "failed to initialize s3.\n");
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
// test put // test put
@ -91,20 +92,17 @@ int32_t s3CheckCfg() {
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) { if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path); fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path); // uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) { if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path); fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path); fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit; TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
} }
taosCloseFile(&fp); taosCloseFile(&fp);
@ -112,7 +110,7 @@ int32_t s3CheckCfg() {
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16); code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) { if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]); fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "put object %s: success.\n\n", objectname[0]); fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
@ -121,7 +119,7 @@ int32_t s3CheckCfg() {
code = s3ListBucket(tsS3BucketName); code = s3ListBucket(tsS3BucketName);
if (code != 0) { if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName); fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName); fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
@ -134,7 +132,7 @@ int32_t s3CheckCfg() {
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock); code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) { if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]); fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
char buf[7] = {0}; char buf[7] = {0};
memcpy(buf, pBlock, c_len); memcpy(buf, pBlock, c_len);
@ -147,14 +145,14 @@ int32_t s3CheckCfg() {
code = s3DeleteObjects(objectname, 1); code = s3DeleteObjects(objectname, 1);
if (code != 0) { if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]); fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit; TAOS_CHECK_GOTO(code, &lino, _exit);
} }
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]); fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End(); s3End();
_exit: _exit:
return code; TAOS_RETURN(code);
} }
static int should_retry() { static int should_retry() {
@ -169,7 +167,7 @@ static int should_retry() {
} }
*/ */
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
@ -202,7 +200,6 @@ static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *prope
} }
static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) {
//(void)callbackData;
TS3SizeCBD *cbd = callbackData; TS3SizeCBD *cbd = callbackData;
if (properties->contentLength > 0) { if (properties->contentLength > 0) {
cbd->content_length = properties->contentLength; cbd->content_length = properties->contentLength;
@ -249,7 +246,7 @@ static int32_t s3ListBucket(char const *bucketname) {
SArray *objectArray = getListByPrefix("s3"); SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) { if (objectArray == NULL) {
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
const char **object_name = TARRAY_DATA(objectArray); const char **object_name = TARRAY_DATA(objectArray);
@ -262,7 +259,7 @@ static int32_t s3ListBucket(char const *bucketname) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code; TAOS_RETURN(code);
} }
typedef struct growbuffer { typedef struct growbuffer {
@ -607,13 +604,13 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} }
return code; TAOS_RETURN(code);
} }
static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name, static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
int64_t contentLength, S3PutProperties *put_prop, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) { put_object_callback_data *data) {
int32_t code = 0; int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength; uint64_t totalContentLength = contentLength;
uint64_t todoContentLength = contentLength; uint64_t todoContentLength = contentLength;
UploadManager manager = {0}; UploadManager manager = {0};
@ -640,6 +637,9 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0; manager.next_etags_pos = 0;
do { do {
S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager); S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
@ -647,8 +647,7 @@ static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, cha
if (manager.upload_id == 0 || manager.status != S3StatusOK) { if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
upload: upload:
@ -672,8 +671,7 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) { if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
contentLength -= chunk_size; contentLength -= chunk_size;
todoContentLength -= chunk_size; todoContentLength -= chunk_size;
@ -686,8 +684,7 @@ upload:
int n; int n;
for (i = 0; i < totalSeq; i++) { for (i = 0; i < totalSeq; i++) {
if (!manager.etags[i]) { if (!manager.etags[i]) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
n = snprintf(buf, sizeof(buf), n = snprintf(buf, sizeof(buf),
"<Part><PartNumber>%d</PartNumber>" "<Part><PartNumber>%d</PartNumber>"
@ -704,11 +701,13 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) { if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
clean: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (manager.upload_id) { if (manager.upload_id) {
taosMemoryFree(manager.upload_id); taosMemoryFree(manager.upload_id);
} }
@ -718,13 +717,13 @@ clean:
growbuffer_destroy(manager.gb); growbuffer_destroy(manager.gb);
taosMemoryFree(manager.etags); taosMemoryFree(manager.etags);
return code; TAOS_RETURN(code);
} }
static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime, static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
char const *object_name, int64_t contentLength, S3PutProperties *put_prop, char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
put_object_callback_data *data) { put_object_callback_data *data) {
int32_t code = 0; int32_t code = 0, lino = 0;
uint64_t totalContentLength = contentLength; uint64_t totalContentLength = contentLength;
// uint64_t todoContentLength = contentLength; // uint64_t todoContentLength = contentLength;
@ -744,13 +743,16 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
SCheckpoint cp = {0}; SCheckpoint cp = {0};
cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart)); cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
if (!cp.parts) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
if (taosCheckExistFile(file_cp_path)) { if (taosCheckExistFile(file_cp_path)) {
if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) { if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
manager.upload_id = strdup(cp.upload_id); manager.upload_id = strdup(cp.upload_id);
need_init_upload = false; need_init_upload = false;
} else { } else {
cos_cp_remove(file_cp_path); TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
} }
} }
@ -763,16 +765,14 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
if (manager.upload_id == 0 || manager.status != S3StatusOK) { if (manager.upload_id == 0 || manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size); cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
} }
if (cos_cp_open(file_cp_path, &cp)) { if (cos_cp_open(file_cp_path, &cp)) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
int part_num = 0; int part_num = 0;
@ -791,6 +791,10 @@ static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const
{&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
if (!manager.etags) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
manager.next_etags_pos = 0; manager.next_etags_pos = 0;
upload: upload:
@ -802,8 +806,7 @@ upload:
if (i > 0 && cp.parts[i - 1].completed) { if (i > 0 && cp.parts[i - 1].completed) {
if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) { if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto clean;
} }
} }
@ -827,25 +830,23 @@ upload:
} while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
if (partData.put_object_data.status != S3StatusOK) { if (partData.put_object_data.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
//(void)cos_cp_dump(&cp); //(void)cos_cp_dump(&cp);
goto clean;
} }
if (!manager.etags[seq - 1]) { if (!manager.etags[seq - 1]) {
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0); cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
(void)cos_cp_dump(&cp); TAOS_CHECK_GOTO(cos_cp_dump(&cp), &lino, _exit);
contentLength -= chunk_size; contentLength -= chunk_size;
// todoContentLength -= chunk_size; // todoContentLength -= chunk_size;
} }
cos_cp_close(cp.thefile); TAOS_CHECK_GOTO(cos_cp_close(cp.thefile), &lino, _exit);
cp.thefile = 0; cp.thefile = 0;
int size = 0; int size = 0;
@ -869,20 +870,23 @@ upload:
} while (S3_status_is_retryable(manager.status) && should_retry()); } while (S3_status_is_retryable(manager.status) && should_retry());
if (manager.status != S3StatusOK) { if (manager.status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
code = TAOS_SYSTEM_ERROR(EIO); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(EIO), &lino, _exit);
goto clean;
} }
cos_cp_remove(file_cp_path); TAOS_CHECK_GOTO(cos_cp_remove(file_cp_path), &lino, _exit);
clean: _exit:
/* /*
if (parts) { if (parts) {
taosMemoryFree(parts); taosMemoryFree(parts);
} }
*/ */
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (cp.thefile) { if (cp.thefile) {
cos_cp_close(cp.thefile); (void)cos_cp_close(cp.thefile);
} }
if (cp.parts) { if (cp.parts) {
taosMemoryFree(cp.parts); taosMemoryFree(cp.parts);
@ -899,7 +903,7 @@ clean:
taosMemoryFree(manager.etags); taosMemoryFree(manager.etags);
growbuffer_destroy(manager.gb); growbuffer_destroy(manager.gb);
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) { int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
@ -917,15 +921,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
put_object_callback_data data = {0}; put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file); uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file); uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -956,7 +958,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
growbuffer_destroy(data.gb); growbuffer_destroy(data.gb);
} }
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
@ -974,22 +976,19 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
put_object_callback_data data = {0}; put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) { if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file); uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
contentLength = size; contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file); uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code; TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
} }
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) { if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD); taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return code;
} }
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength = data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
@ -1016,7 +1015,7 @@ int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int
growbuffer_destroy(data.gb); growbuffer_destroy(data.gb);
} }
return code; TAOS_RETURN(code);
} }
typedef struct list_bucket_callback_data { typedef struct list_bucket_callback_data {
@ -1130,11 +1129,11 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1; code = TSDB_CODE_FAILED;
} }
} }
return code; TAOS_RETURN(code);
} }
void s3DeleteObjectsByPrefix(const char *prefix) { void s3DeleteObjectsByPrefix(const char *prefix) {
@ -1187,17 +1186,19 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
if (check && cbd.buf_pos != size) { if (check && cbd.buf_pos != size) {
uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO);
TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
*ppBlock = cbd.buf; *ppBlock = cbd.buf;
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) {
@ -1218,9 +1219,8 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); uError("[s3] open file error, errno:%d, fileName:%s", TAOS_SYSTEM_ERROR(errno), fileName);
uError("[s3] open file error, errno:%d, fileName:%s", terrno, fileName); TAOS_RETURN(TAOS_SYSTEM_ERROR(errno));
return -1;
} }
TS3GetData cbd = {0}; TS3GetData cbd = {0};
@ -1232,16 +1232,17 @@ int32_t s3GetObjectToFile(const char *object_name, const char *fileName) {
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
return 0;
TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
SArray *objectArray = getListByPrefix(prefix); SArray *objectArray = getListByPrefix(prefix);
if (objectArray == NULL) return -1; if (objectArray == NULL) TAOS_RETURN(TSDB_CODE_FAILED);
for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
char *object = taosArrayGetP(objectArray, i); char *object = taosArrayGetP(objectArray, i);
@ -1255,7 +1256,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
} }
if (s3GetObjectToFile(object, fileName) != 0) { if (s3GetObjectToFile(object, fileName) != 0) {
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
} }
taosArrayDestroyEx(objectArray, s3FreeObjectKey); taosArrayDestroyEx(objectArray, s3FreeObjectKey);
@ -1279,7 +1280,7 @@ long s3Size(const char *object_name) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
return -1; TAOS_RETURN(TSDB_CODE_FAILED);
} }
size = cbd.content_length; size = cbd.content_length;
@ -1306,7 +1307,7 @@ int32_t s3Init() {
// set log output, default stderr // set log output, default stderr
cos_log_set_output(NULL); cos_log_set_output(NULL);
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
void s3CleanUp() { cos_http_io_deinitialize(); } void s3CleanUp() { cos_http_io_deinitialize(); }
@ -1363,10 +1364,10 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
cos_pool_destroy(p); cos_pool_destroy(p);
if (s->code != 200) { if (s->code != 200) {
return code = s->code; TAOS_RETURN(s->code);
} }
return code; TAOS_RETURN(code);
} }
int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) { int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
@ -1405,10 +1406,10 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_
cos_pool_destroy(p); cos_pool_destroy(p);
if (s->code != 200) { if (s->code != 200) {
return code = s->code; TAOS_RETURN(s->code);
} }
return code; TAOS_RETURN(code);
} }
void s3DeleteObjectsByPrefix(const char *prefix_str) { void s3DeleteObjectsByPrefix(const char *prefix_str) {
@ -1465,7 +1466,7 @@ int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_warn_log("delete objects failed\n"); cos_warn_log("delete objects failed\n");
} }
return 0; TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
bool s3Exists(const char *object_name) { bool s3Exists(const char *object_name) {
@ -1547,7 +1548,7 @@ bool s3Get(const char *object_name, const char *path) {
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
(void)check; (void)check;
int32_t code = 0; int32_t code = 0, lino = 0;
cos_pool_t *p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_status_t *s = NULL; cos_status_t *s = NULL;
@ -1588,10 +1589,9 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers); s = cos_get_object_to_buffer(options, &bucket, &object, headers, NULL, &download_buffer, &resp_headers);
log_status(s); log_status(s);
if (!cos_status_is_ok(s)) { if (!cos_status_is_ok(s)) {
vError("s3: %d(%s)", s->code, s->error_msg); uError("s3: %d(%s)", s->code, s->error_msg);
vError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno)); uError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
code = TAOS_SYSTEM_ERROR(EIO); TAOS_RETURN(TAOS_SYSTEM_ERROR(EIO));
return code;
} }
// print_headers(resp_headers); // print_headers(resp_headers);
@ -1601,6 +1601,10 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); } cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { len += cos_buf_size(content); }
// char *buf = cos_pcalloc(p, (apr_size_t)(len + 1)); // char *buf = cos_pcalloc(p, (apr_size_t)(len + 1));
char *buf = taosMemoryCalloc(1, (apr_size_t)(len)); char *buf = taosMemoryCalloc(1, (apr_size_t)(len));
if (!buf) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
// buf[len] = '\0'; // buf[len] = '\0';
cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) { cos_list_for_each_entry(cos_buf_t, content, &download_buffer, node) {
size = cos_buf_size(content); size = cos_buf_size(content);
@ -1609,12 +1613,13 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_
} }
// cos_warn_log("Download data=%s", buf); // cos_warn_log("Download data=%s", buf);
_exit:
//销毁内存池 //销毁内存池
cos_pool_destroy(p); cos_pool_destroy(p);
*ppBlock = buf; *ppBlock = buf;
return code; TAOS_RETURN(code);
} }
typedef struct { typedef struct {

172
source/common/src/cos_cp.c
View File

@ -9,42 +9,35 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH); TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
if (!fd) { if (!fd) {
code = TAOS_SYSTEM_ERROR(errno); uError("%s Failed to open %s", __func__, cp_path);
uError("ERROR: %s Failed to open %s", __func__, cp_path); TAOS_CHECK_RETURN(TAOS_SYSTEM_ERROR(errno));
return code;
} }
checkpoint->thefile = fd; checkpoint->thefile = fd;
return code; TAOS_RETURN(code);
} }
void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } int32_t cos_cp_close(TdFilePtr fd) { return taosCloseFile(&fd); }
void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); } int32_t cos_cp_remove(char const* filepath) { return taosRemoveFile(filepath); }
static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
int32_t code = 0; int32_t code = 0, lino = 0;
cJSON const* item2 = NULL; cJSON const* item2 = NULL;
cJSON* json = cJSON_Parse(cp_body); cJSON* json = cJSON_Parse(cp_body);
if (NULL == json) { if (NULL == json) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
} }
cJSON const* item = cJSON_GetObjectItem(json, "ver"); cJSON const* item = cJSON_GetObjectItem(json, "ver");
if (!cJSON_IsNumber(item) || item->valuedouble != 1) { if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
goto _exit;
} }
item = cJSON_GetObjectItem(json, "type"); item = cJSON_GetObjectItem(json, "type");
if (!cJSON_IsNumber(item)) { if (!cJSON_IsNumber(item)) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to parse json", __func__);
goto _exit;
} }
cp->cp_type = item->valuedouble; cp->cp_type = item->valuedouble;
@ -155,40 +148,36 @@ static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
} }
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (json) cJSON_Delete(json); if (json) cJSON_Delete(json);
if (cp_body) taosMemoryFree(cp_body); if (cp_body) taosMemoryFree(cp_body);
TAOS_RETURN(code);
return code;
} }
int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) { int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
int32_t code = 0; int32_t code = 0, lino = 0;
char* cp_body = NULL;
TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ); TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
if (!fd) { if (!fd) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
uError("ERROR: %s Failed to open %s", __func__, filepath);
goto _exit;
} }
int64_t size = -1; int64_t size = -1;
code = taosStatFile(filepath, &size, NULL, NULL); TAOS_CHECK_GOTO(taosStatFile(filepath, &size, NULL, NULL), &lino, _exit);
if (code) {
uError("ERROR: %s Failed to stat %s", __func__, filepath);
goto _exit;
}
char* cp_body = taosMemoryMalloc(size + 1); cp_body = taosMemoryMalloc(size + 1);
if (!cp_body) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
}
int64_t n = taosReadFile(fd, cp_body, size); int64_t n = taosReadFile(fd, cp_body, size);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
uError("ERROR: %s Failed to read %s", __func__, filepath);
goto _exit;
} else if (n != size) { } else if (n != size) {
code = TSDB_CODE_FILE_CORRUPTED; TAOS_CHECK_GOTO(TSDB_CODE_FILE_CORRUPTED, &lino, _exit);
uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
goto _exit;
} }
taosCloseFile(&fd); taosCloseFile(&fd);
cp_body[size] = '\0'; cp_body[size] = '\0';
@ -196,184 +185,161 @@ int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
return cos_cp_parse_body(cp_body, checkpoint); return cos_cp_parse_body(cp_body, checkpoint);
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
if (fd) { if (fd) {
taosCloseFile(&fd); taosCloseFile(&fd);
} }
if (cp_body) { if (cp_body) {
taosMemoryFree(cp_body); taosMemoryFree(cp_body);
} }
TAOS_RETURN(code);
return code;
} }
static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) { static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
int32_t code = 0; int32_t code = 0, lino = 0;
char* data = cJSON_PrintUnformatted(json); char* data = cJSON_PrintUnformatted(json);
if (NULL == data) { if (!data) {
return TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
} }
TdFilePtr fp = checkpoint->thefile; TdFilePtr fp = checkpoint->thefile;
if (taosFtruncateFile(fp, 0) < 0) { if (taosFtruncateFile(fp, 0) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosLSeekFile(fp, 0, SEEK_SET) < 0) { if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosWriteFile(fp, data, strlen(data)) < 0) { if (taosWriteFile(fp, data, strlen(data)) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
if (taosFsyncFile(fp) < 0) { if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno); TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &lino, _exit);
goto _exit;
} }
_exit: _exit:
if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(data); taosMemoryFree(data);
return code; TAOS_RETURN(code);
} }
int32_t cos_cp_dump(SCheckpoint* cp) { int32_t cos_cp_dump(SCheckpoint* cp) {
int32_t code = 0; int32_t code = 0, lino = 0;
int32_t lino = 0;
cJSON* ojson = NULL; cJSON* ojson = NULL;
cJSON* json = cJSON_CreateObject(); cJSON* json = cJSON_CreateObject();
if (!json) return TSDB_CODE_OUT_OF_MEMORY; if (!json) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) { if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) { if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) { if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) { if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (COS_CP_TYPE_UPLOAD == cp->cp_type) { if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "file"); ojson = cJSON_AddObjectToObject(json, "file");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) { if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) { if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) { if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) { } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
ojson = cJSON_AddObjectToObject(json, "object"); ojson = cJSON_AddObjectToObject(json, "object");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) { if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) { if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) { if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
ojson = cJSON_AddObjectToObject(json, "cpparts"); ojson = cJSON_AddObjectToObject(json, "cpparts");
if (!ojson) { if (!ojson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) { if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) { if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts"); cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
if (!ajson) { if (!ajson) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
for (int i = 0; i < cp->part_num; ++i) { for (int i = 0; i < cp->part_num; ++i) {
cJSON* item = cJSON_CreateObject(); cJSON* item = cJSON_CreateObject();
if (!item) { if (!item) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
cJSON_AddItemToArray(ajson, item); cJSON_AddItemToArray(ajson, item);
if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) { if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) { if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) { if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) { if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) { if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) { if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
code = TSDB_CODE_OUT_OF_MEMORY; TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _exit);
TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = cos_cp_save_json(json, cp); code = cos_cp_save_json(json, cp);
TSDB_CHECK_CODE(code, lino, _exit); TAOS_CHECK_GOTO(code, &lino, _exit);
_exit: _exit:
if (code) { if (code) {
uError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
} }
cJSON_Delete(json); cJSON_Delete(json);
return code; TAOS_RETURN(code);
} }
void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {} void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}

11
source/common/src/tdatablock.c
View File

@ -615,6 +615,10 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex)
int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex;
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index);
if (pColInfoData == NULL) {
return 0;
}
if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) {
return 0; return 0;
} }
@ -1514,7 +1518,7 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
taosArrayDestroy(pBlock->pDataBlock); taosArrayDestroy(pBlock->pDataBlock);
pBlock->pDataBlock = NULL; pBlock->pDataBlock = NULL;
taosMemoryFreeClear(pBlock->pBlockAgg); taosMemoryFreeClear(pBlock->pBlockAgg);
memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); memset(&pBlock->info, 0, sizeof(SDataBlockInfo));
} }
@ -2488,7 +2492,10 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbFullName, uint64_t groupId, cha
RandTableName rname = { RandTableName rname = {
.tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname}; .tags = tags, .stbFullName = stbFullName, .stbFullNameLen = strlen(stbFullName), .ctbShortName = cname};
buildChildTableName(&rname); int32_t code = buildChildTableName(&rname);
if(code != TSDB_CODE_SUCCESS){
return code;
}
taosArrayDestroy(tags); taosArrayDestroy(tags);
if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) {

76
source/common/src/tdataformat.c
View File

@ -305,11 +305,11 @@ static int32_t tRowBuildTupleRow(SArray *aColVal, const SRowBuildScanInfo *sinfo
*(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize; *(int32_t *)(fixed + schema->columns[i].offset) = varlen - fixed - sinfo->tupleFixedSize;
varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData); varlen += tPutU32v(varlen, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData) { if (colValArray[colValIndex].value.nData) {
memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); (void)memcpy(varlen, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
varlen += colValArray[colValIndex].value.nData; varlen += colValArray[colValIndex].value.nData;
} }
} else { } else {
memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val, (void)memcpy(fixed + schema->columns[i].offset, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes); tDataTypes[schema->columns[i].type].bytes);
} }
} else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL } else if (COL_VAL_IS_NULL(&colValArray[colValIndex])) { // NULL
@ -384,12 +384,12 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData);
if (colValArray[colValIndex].value.nData > 0) { if (colValArray[colValIndex].value.nData > 0) {
memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); (void)memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData);
} }
payloadSize += colValArray[colValIndex].value.nData; payloadSize += colValArray[colValIndex].value.nData;
} else { } else {
payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid);
memcpy(payload + payloadSize, &colValArray[colValIndex].value.val, (void)memcpy(payload + payloadSize, &colValArray[colValIndex].value.val,
tDataTypes[schema->columns[i].type].bytes); tDataTypes[schema->columns[i].type].bytes);
payloadSize += tDataTypes[schema->columns[i].type].bytes; payloadSize += tDataTypes[schema->columns[i].type].bytes;
} }
@ -475,7 +475,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
value.nData = infos[iInfo].bind->length[iRow]; value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow; value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else { } else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow, (void)memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length); infos[iInfo].bind->buffer_length);
} }
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value); colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
@ -509,7 +509,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->cid = pTColumn->colId; pColVal->cid = pTColumn->colId;
pColVal->value.type = pTColumn->type; pColVal->value.type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE; pColVal->flag = CV_FLAG_VALUE;
memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); (void)memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY));
return 0; return 0;
} }
@ -573,7 +573,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = NULL; pColVal->value.pData = NULL;
} }
} else { } else {
memcpy(&pColVal->value.val, pData, pTColumn->bytes); (void)memcpy(&pColVal->value.val, pData, pTColumn->bytes);
} }
} }
return 0; return 0;
@ -624,7 +624,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset); pColVal->value.pData = varlen + *(int32_t *)(fixed + pTColumn->offset);
pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData); pColVal->value.pData += tGetU32v(pColVal->value.pData, &pColVal->value.nData);
} else { } else {
memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]); (void)memcpy(&pColVal->value.val, fixed + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
} }
} }
@ -861,7 +861,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.cid = pTColumn->colId; pIter->cv.cid = pTColumn->colId;
pIter->cv.value.type = pTColumn->type; pIter->cv.value.type = pTColumn->type;
pIter->cv.flag = CV_FLAG_VALUE; pIter->cv.flag = CV_FLAG_VALUE;
memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY)); (void)memcpy(&pIter->cv.value.val, &pIter->pRow->ts, sizeof(TSKEY));
goto _exit; goto _exit;
} }
@ -906,7 +906,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL; pIter->cv.value.pData = NULL;
} }
} else { } else {
memcpy(&pIter->cv.value.val, pData, pTColumn->bytes); (void)memcpy(&pIter->cv.value.val, pData, pTColumn->bytes);
} }
} }
@ -965,7 +965,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.value.pData = NULL; pIter->cv.value.pData = NULL;
} }
} else { } else {
memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); (void)memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]);
} }
goto _exit; goto _exit;
} }
@ -1288,7 +1288,7 @@ void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
key->pks[i].pData = tdata; key->pks[i].pData = tdata;
key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData); key->pks[i].pData += tGetU32v(key->pks[i].pData, &key->pks[i].nData);
} else { } else {
memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes); (void)memcpy(&key->pks[i].val, tdata, tDataTypes[indices[i].type].bytes);
} }
} }
} }
@ -1378,7 +1378,7 @@ FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
return 0; return 0;
} }
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
pDst->ts = pSrc->ts; pDst->ts = pSrc->ts;
pDst->numOfPKs = pSrc->numOfPKs; pDst->numOfPKs = pSrc->numOfPKs;
@ -1392,12 +1392,10 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) {
} else { } else {
pVal->nData = pSrc->pks[i].nData; pVal->nData = pSrc->pks[i].nData;
ASSERT(pSrc->pks[i].pData != NULL); ASSERT(pSrc->pks[i].pData != NULL);
memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); (void)memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData);
} }
} }
} }
return TSDB_CODE_SUCCESS;
} }
// STag ======================================== // STag ========================================
@ -1528,7 +1526,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
} else { } else {
p = p ? p + n : p; p = p ? p + n : p;
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
if (p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); if (p) (void)memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes);
} }
return n; return n;
@ -1550,7 +1548,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
if (IS_VAR_DATA_TYPE(pTagVal->type)) { if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData);
} else { } else {
memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); (void)memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
} }
@ -1661,7 +1659,7 @@ char *tTagValToData(const STagVal *value, bool isJson) {
} }
varDataLen(data + typeBytes) = value->nData; varDataLen(data + typeBytes) = value->nData;
memcpy(varDataVal(data + typeBytes), value->pData, value->nData); (void)memcpy(varDataVal(data + typeBytes), value->pData, value->nData);
} else { } else {
data = ((char *)&(value->i64)) - typeBytes; // json with type data = ((char *)&(value->i64)) - typeBytes; // json with type
} }
@ -1713,7 +1711,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) {
} else if (c > 0) { } else if (c > 0) {
lidx = midx + 1; lidx = midx + 1;
} else { } else {
memcpy(pTagVal, &tv, sizeof(tv)); (void)memcpy(pTagVal, &tv, sizeof(tv));
return true; return true;
} }
} }
@ -1892,7 +1890,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
if (nData) { if (nData) {
code = tRealloc(&pColData->pData, pColData->nData + nData); code = tRealloc(&pColData->pData, pColData->nData + nData);
if (code) goto _exit; if (code) goto _exit;
memcpy(pColData->pData + pColData->nData, pData, nData); (void)memcpy(pColData->pData + pColData->nData, pData, nData);
pColData->nData += nData; pColData->nData += nData;
} }
} else { } else {
@ -1900,7 +1898,7 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData,
code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
if (code) goto _exit; if (code) goto _exit;
if (pData) { if (pData) {
memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); (void)memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]);
} else { } else {
memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]);
} }
@ -2594,7 +2592,7 @@ static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SCo
} }
value.pData = pColData->pData + pColData->aOffset[iVal]; value.pData = pColData->pData + pColData->aOffset[iVal];
} else { } else {
memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes); (void)memcpy(&value.val, pColData->pData + tDataTypes[pColData->type].bytes * iVal, tDataTypes[pColData->type].bytes);
} }
*pColVal = COL_VAL_VALUE(pColData->cid, value); *pColVal = COL_VAL_VALUE(pColData->cid, value);
} }
@ -2692,7 +2690,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal));
break; break;
case (HAS_VALUE | HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NULL | HAS_NONE):
pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal));
@ -2700,7 +2698,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); (void)memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal));
break; break;
default: default:
pColData->pBitMap = NULL; pColData->pBitMap = NULL;
@ -2714,7 +2712,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
} }
memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); (void)memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2);
} else { } else {
pColData->aOffset = NULL; pColData->aOffset = NULL;
} }
@ -2727,7 +2725,7 @@ int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMall
goto _exit; goto _exit;
} }
memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); (void)memcpy(pColData->pData, pColDataFrom->pData, pColData->nData);
} else { } else {
pColData->pData = NULL; pColData->pData = NULL;
} }
@ -3119,10 +3117,10 @@ static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SCo
pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData;
} }
memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], (void)memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow],
nData); nData);
} else { } else {
memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], (void)memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow],
&pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]);
} }
return code; return code;
@ -3361,7 +3359,7 @@ static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /
ASSERT(0); ASSERT(0);
} else { } else {
if (iv != iStart) { if (iv != iStart) {
memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], (void)memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart],
&pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]);
} }
memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)],
@ -3579,11 +3577,11 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NONE):
case (HAS_VALUE | HAS_NULL): case (HAS_VALUE | HAS_NULL):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal));
n += BIT1_SIZE(pColData->nVal); n += BIT1_SIZE(pColData->nVal);
break; break;
case (HAS_VALUE | HAS_NULL | HAS_NONE): case (HAS_VALUE | HAS_NULL | HAS_NONE):
if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); if (pBuf) (void)memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal));
n += BIT2_SIZE(pColData->nVal); n += BIT2_SIZE(pColData->nVal);
break; break;
default: default:
@ -3593,14 +3591,14 @@ static int32_t tPutColDataVersion0(uint8_t *pBuf, SColData *pColData) {
// value // value
if (pColData->flag & HAS_VALUE) { if (pColData->flag & HAS_VALUE) {
if (IS_VAR_DATA_TYPE(pColData->type)) { if (IS_VAR_DATA_TYPE(pColData->type)) {
if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); if (pBuf) (void)memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
n += (pColData->nVal << 2); n += (pColData->nVal << 2);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData);
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData; n += pColData->nData;
} else { } else {
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); if (pBuf) (void)memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData; n += pColData->nData;
} }
} }
@ -4348,7 +4346,7 @@ int32_t tCompressData(void *input, // input
ASSERT(outputSize >= extraSizeNeeded); ASSERT(outputSize >= extraSizeNeeded);
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
memcpy(output, input, info->originalSize); (void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize; info->compressedSize = info->originalSize;
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
@ -4385,7 +4383,7 @@ int32_t tCompressData(void *input, // input
} else { } else {
DEFINE_VAR(info->cmprAlg) DEFINE_VAR(info->cmprAlg)
if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) { if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) {
memcpy(output, input, info->originalSize); (void)memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize; info->compressedSize = info->originalSize;
return 0; return 0;
} }
@ -4431,7 +4429,7 @@ int32_t tDecompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize); ASSERT(info->compressedSize == info->originalSize);
memcpy(output, input, info->compressedSize); (void)memcpy(output, input, info->compressedSize);
} else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
@ -4468,7 +4466,7 @@ int32_t tDecompressData(void *input, // input
} else { } else {
DEFINE_VAR(info->cmprAlg); DEFINE_VAR(info->cmprAlg);
if (l1 == L1_DISABLED && l2 == L2_DISABLED) { if (l1 == L1_DISABLED && l2 == L2_DISABLED) {
memcpy(output, input, info->compressedSize); (void)memcpy(output, input, info->compressedSize);
return 0; return 0;
} }
SBuffer local; SBuffer local;

2
source/common/src/tglobal.c
View File

@ -504,7 +504,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1; return -1;

167
source/common/src/tmisce.c
View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tmisce.h" #include "tmisce.h"
#include "tdatablock.h"
#include "tglobal.h" #include "tglobal.h"
#include "tjson.h" #include "tjson.h"
#include "tdatablock.h"
int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = 0; pEp->port = 0;
@ -34,18 +34,26 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
pEp->port = tsServerPort; pEp->port = tsServerPort;
} }
if (pEp->port <= 0) {
return TSDB_CODE_INVALID_PARA;
}
return 0; return 0;
} }
void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { int32_t addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) {
if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) {
return; return TSDB_CODE_INVALID_PARA;
} }
int32_t index = pEpSet->numOfEps; int32_t index = pEpSet->numOfEps;
if (index >= sizeof(pEpSet->eps) / sizeof(pEpSet->eps[0])) {
return TSDB_CODE_OUT_OF_RANGE;
}
tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn));
pEpSet->eps[index].port = port; pEpSet->eps[index].port = port;
pEpSet->numOfEps += 1; pEpSet->numOfEps += 1;
return 0;
} }
bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) {
@ -131,75 +139,96 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet) {
return ep; return ep;
} }
int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t bufLen) { int32_t epsetToStr(const SEpSet* pEpSet, char* pBuf, int32_t cap) {
int len = snprintf(pBuf, bufLen, "epset:{"); int32_t ret = 0;
if (len < 0) { int32_t nwrite = 0;
return -1;
}
for (int _i = 0; (_i < pEpSet->numOfEps) && (bufLen > len); _i++) { nwrite = snprintf(pBuf + nwrite, cap, "epset:{");
if (nwrite <= 0 || nwrite >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
cap -= nwrite;
for (int _i = 0; (_i < pEpSet->numOfEps) && (cap > 0); _i++) {
int32_t ret = 0; int32_t ret = 0;
if (_i == pEpSet->numOfEps - 1) { if (_i == pEpSet->numOfEps - 1) {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} else { } else {
ret = snprintf(pBuf + len, bufLen - len, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port); ret = snprintf(pBuf + nwrite, cap, "%d. %s:%d, ", _i, pEpSet->eps[_i].fqdn, pEpSet->eps[_i].port);
} }
if (ret < 0) { if (ret <= 0 || ret >= cap) {
return -1; return TSDB_CODE_OUT_OF_BUFFER;
} }
len += ret; nwrite += ret;
cap -= ret;
} }
if (len < bufLen) { if (cap <= 0) {
/*len += */snprintf(pBuf + len, bufLen - len, "}, inUse:%d", pEpSet->inUse); return TSDB_CODE_OUT_OF_BUFFER;
} }
ret = snprintf(pBuf + nwrite, cap, "}, inUse:%d", pEpSet->inUse);
if (ret <= 0 || ret >= cap) {
return TSDB_CODE_OUT_OF_BUFFER;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) { int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime) {
SJson* pJson = tjsonCreateObject(); int32_t code = 0;
if (pJson == NULL) return -1; SJson* pJson = tjsonCreateObject();
if (pJson == NULL) return TSDB_CODE_OUT_OF_MEMORY;
char tmp[4096] = {0}; char tmp[4096] = {0};
tjsonAddDoubleToObject(pJson, "reportVersion", 1); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "reportVersion", 1), NULL, _exit);
tjsonAddIntegerToObject(pJson, "clusterId", clusterId); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "clusterId", clusterId), NULL, _exit);
tjsonAddIntegerToObject(pJson, "startTime", startTime); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "startTime", startTime), NULL, _exit);
// Do NOT invoke the taosGetFqdn here. // Do NOT invoke the taosGetFqdn here.
// this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked // this function may be invoked when memory exception occurs,so we should assume that it is running in a memory locked
// environment. The lock operation by taosGetFqdn may cause this program deadlock. // environment. The lock operation by taosGetFqdn may cause this program deadlock.
tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "fqdn", tsLocalFqdn), NULL, _exit);
tjsonAddIntegerToObject(pJson, "pid", taosGetPId()); TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "pid", taosGetPId()), NULL, _exit);
taosGetAppName(tmp, NULL); code = taosGetAppName(tmp, NULL);
tjsonAddStringToObject(pJson, "appName", tmp); if (code != 0) {
code = TAOS_SYSTEM_ERROR(errno);
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "appName", tmp), NULL, _exit);
if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) { if (taosGetOsReleaseName(tmp, NULL, NULL, sizeof(tmp)) == 0) {
tjsonAddStringToObject(pJson, "os", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "os", tmp), NULL, _exit);
} else {
// do nothing
} }
float numOfCores = 0; float numOfCores = 0;
if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) { if (taosGetCpuInfo(tmp, sizeof(tmp), &numOfCores) == 0) {
tjsonAddStringToObject(pJson, "cpuModel", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "cpuModel", tmp), NULL, _exit);
tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", numOfCores), NULL, _exit);
} else { } else {
tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores); TAOS_CHECK_GOTO(tjsonAddDoubleToObject(pJson, "numOfCpu", tsNumOfCores), NULL, _exit);
} }
snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB); int32_t nBytes = snprintf(tmp, sizeof(tmp), "%" PRId64 " kB", tsTotalMemoryKB);
tjsonAddStringToObject(pJson, "memory", tmp); if (nBytes <= 9 || nBytes >= sizeof(tmp)) {
TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_RANGE, NULL, _exit);
}
TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "memory", tmp), NULL, _exit);
tjsonAddStringToObject(pJson, "version", version); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "version", version), NULL, _exit);
tjsonAddStringToObject(pJson, "buildInfo", buildinfo); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "buildInfo", buildinfo), NULL, _exit);
tjsonAddStringToObject(pJson, "gitInfo", gitinfo);
tjsonAddIntegerToObject(pJson, "crashSig", signum); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "gitInfo", gitinfo), NULL, _exit);
tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs());
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashSig", signum), NULL, _exit);
TAOS_CHECK_GOTO(tjsonAddIntegerToObject(pJson, "crashTs", taosGetTimestampUs()), NULL, _exit);
#ifdef _TD_DARWIN_64 #ifdef _TD_DARWIN_64
taosLogTraceToBuf(tmp, sizeof(tmp), 4); taosLogTraceToBuf(tmp, sizeof(tmp), 4);
@ -209,26 +238,47 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t
taosLogTraceToBuf(tmp, sizeof(tmp), 8); taosLogTraceToBuf(tmp, sizeof(tmp), 8);
#endif #endif
tjsonAddStringToObject(pJson, "stackInfo", tmp); TAOS_CHECK_GOTO(tjsonAddStringToObject(pJson, "stackInfo", tmp), NULL, _exit);
char* pCont = tjsonToString(pJson); char* pCont = tjsonToString(pJson);
if (pCont == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
goto _exit;
}
tjsonDelete(pJson); tjsonDelete(pJson);
*pMsg = pCont; *pMsg = pCont;
pJson = NULL;
return TSDB_CODE_SUCCESS; _exit:
tjsonDelete(pJson);
TAOS_RETURN(code);
} }
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) { int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
SConfig* pConf = taosGetCfg(); int32_t code = 0;
SConfig* pConf = taosGetCfg();
if (pConf == NULL) {
return TSDB_CODE_INVALID_CFG;
}
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t col = startCol; int32_t col = startCol;
SConfigItem* pItem = NULL; SConfigItem* pItem = NULL;
SConfigIter* pIter = NULL;
blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)); int8_t locked = 0;
SConfigIter* pIter = cfgCreateIter(pConf);
TAOS_CHECK_GOTO(blockDataEnsureCapacity(pBlock, cfgGetSize(pConf)), NULL, _exit);
pIter = cfgCreateIter(pConf);
if (pIter == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
cfgLock(pConf); cfgLock(pConf);
locked = 1;
while ((pItem = cfgNextIter(pIter)) != NULL) { while ((pItem = cfgNextIter(pIter)) != NULL) {
col = startCol; col = startCol;
@ -236,29 +286,44 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
// GRANT_CFG_SKIP; // GRANT_CFG_SKIP;
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, name, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, name, false), NULL, _exit);
char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t valueLen = 0; int32_t valueLen = 0;
cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen);
varDataSetLen(value, valueLen); varDataSetLen(value, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++); pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, value, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, value, false), NULL, _exit);
char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0}; char scope[TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE] = {0};
cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen); cfgDumpItemScope(pItem, &scope[VARSTR_HEADER_SIZE], TSDB_CONFIG_SCOPE_LEN, &valueLen);
varDataSetLen(scope, valueLen); varDataSetLen(scope, valueLen);
pColInfo = taosArrayGet(pBlock->pDataBlock, col++); pColInfo = taosArrayGet(pBlock->pDataBlock, col++);
colDataSetVal(pColInfo, numOfRows, scope, false); if (pColInfo == NULL) {
code = TSDB_CODE_OUT_OF_RANGE;
TAOS_CHECK_GOTO(code, NULL, _exit);
}
TAOS_CHECK_GOTO(colDataSetVal(pColInfo, numOfRows, scope, false), NULL, _exit);
numOfRows++; numOfRows++;
} }
cfgUnLock(pConf);
pBlock->info.rows = numOfRows; pBlock->info.rows = numOfRows;
_exit:
if (locked) cfgUnLock(pConf);
cfgDestroyIter(pIter); cfgDestroyIter(pIter);
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }

18
source/common/src/tname.c
View File

@ -98,7 +98,7 @@ SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName
int32_t tNameExtractFullName(const SName* name, char* dst) { int32_t tNameExtractFullName(const SName* name, char* dst) {
// invalid full name format, abort // invalid full name format, abort
if (!tNameIsValid(name)) { if (!tNameIsValid(name)) {
return -1; return TSDB_CODE_INVALID_PARA;
} }
int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname);
@ -154,8 +154,7 @@ int32_t tNameGetDbName(const SName* name, char* dst) {
const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; } const char* tNameGetDbNameP(const SName* name) { return &name->dbname[0]; }
int32_t tNameGetFullDbName(const SName* name, char* dst) { int32_t tNameGetFullDbName(const SName* name, char* dst) {
snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname); return snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname);
return 0;
} }
bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; }
@ -293,17 +292,19 @@ static int compareKv(const void* p1, const void* p2) {
/* /*
* use stable name and tags to grearate child table name * use stable name and tags to grearate child table name
*/ */
void buildChildTableName(RandTableName* rName) { int32_t buildChildTableName(RandTableName* rName) {
SStringBuilder sb = {0}; SStringBuilder sb = {0};
taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen);
if (sb.buf == NULL) { if (sb.buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return;
} }
taosArraySort(rName->tags, compareKv); taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ','); taosStringBuilderAppendChar(&sb, ',');
SSmlKv* tagKv = taosArrayGet(rName->tags, j); SSmlKv* tagKv = taosArrayGet(rName->tags, j);
if(tagKv == NULL) {
return TSDB_CODE_SML_INVALID_DATA;
}
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '='); taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) { if (IS_VAR_DATA_TYPE(tagKv->type)) {
@ -323,8 +324,9 @@ void buildChildTableName(RandTableName* rName) {
rName->ctbShortName[0] = 't'; rName->ctbShortName[0] = 't';
rName->ctbShortName[1] = '_'; rName->ctbShortName[1] = '_';
for (int i = 0; i < 16; i++) { for (int i = 0; i < 16; i++) {
sprintf(temp, "%02x", context.digest[i]); (void)sprintf(temp, "%02x", context.digest[i]);
strcat(rName->ctbShortName, temp); (void)strcat(rName->ctbShortName, temp);
} }
taosStringBuilderDestroy(&sb); taosStringBuilderDestroy(&sb);
return TSDB_CODE_SUCCESS;
} }

12
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -153,6 +153,8 @@ int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
int32_t code;
SSingleWorker *pWorker = NULL; SSingleWorker *pWorker = NULL;
switch (qtype) { switch (qtype) {
case WRITE_QUEUE: case WRITE_QUEUE:
@ -181,13 +183,15 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
} }
if (pWorker == NULL) return -1; if (pWorker == NULL) return -1;
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); SRpcMsg *pMsg;
if (pMsg == NULL) return -1; code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL; pRpc->pCont = NULL;
dTrace("msg:%p, is created and will put into %s queue, type:%s len:%d", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType), pRpc->contLen); dTrace("msg:%p, is created and will put into %s queue, type:%s len:%d", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType),
int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); pRpc->contLen);
code = mmPutMsgToWorker(pMgmt, pWorker, pMsg);
if (code != 0) { if (code != 0) {
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);

7
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -58,8 +58,11 @@ int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); int32_t code;
if (pMsg == NULL) return -1; SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) return code;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL; pRpc->pCont = NULL;

7
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -126,8 +126,11 @@ void smStopWorker(SSnodeMgmt *pMgmt) {
} }
int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); int32_t code;
if (pMsg == NULL) { SRpcMsg *pMsg;
code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return -1;

8
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -311,6 +311,7 @@ int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
int32_t code;
if (pRpc->contLen < sizeof(SMsgHead)) { if (pRpc->contLen < sizeof(SMsgHead)) {
dError("invalid rpc msg with no msg head at pCont. pRpc:%p, type:%s, len:%d", pRpc, TMSG_INFO(pRpc->msgType), dError("invalid rpc msg with no msg head at pCont. pRpc:%p, type:%s, len:%d", pRpc, TMSG_INFO(pRpc->msgType),
pRpc->contLen); pRpc->contLen);
@ -320,8 +321,9 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
} }
EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM; EQItype itype = APPLY_QUEUE == qtype ? DEF_QITEM : RPC_QITEM;
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen); SRpcMsg *pMsg;
if (pMsg == NULL) { code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg);
if (code) {
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
pRpc->pCont = NULL; pRpc->pCont = NULL;
return -1; return -1;
@ -335,7 +337,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
pRpc->pCont = NULL; pRpc->pCont = NULL;
int32_t code = vmPutMsgToQueue(pMgmt, pMsg, qtype); code = vmPutMsgToQueue(pMgmt, pMsg, qtype);
if (code != 0) { if (code != 0) {
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);

4
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -210,8 +210,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
pRpc->info.wrapper = pWrapper; pRpc->info.wrapper = pWrapper;
EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsRpcQueueMemoryUsed EQItype itype = IsReq(pRpc) ? RPC_QITEM : DEF_QITEM; // rsp msg is not restricted by tsRpcQueueMemoryUsed
pMsg = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen); code = taosAllocateQitem(sizeof(SRpcMsg), itype, pRpc->contLen, (void **)&pMsg);
if (pMsg == NULL) goto _OVER; if (code) goto _OVER;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle, dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle,

60
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -54,6 +54,7 @@ int32_t mndInitAcct(SMnode *pMnode) {
void mndCleanupAcct(SMnode *pMnode) {} void mndCleanupAcct(SMnode *pMnode) {}
static int32_t mndCreateDefaultAcct(SMnode *pMnode) { static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
int32_t code = 0;
SAcctObj acctObj = {0}; SAcctObj acctObj = {0};
tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN);
acctObj.createdTime = taosGetTimestampMs(); acctObj.createdTime = taosGetTimestampMs();
@ -76,7 +77,10 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
}; };
SSdbRaw *pRaw = mndAcctActionEncode(&acctObj); SSdbRaw *pRaw = mndAcctActionEncode(&acctObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw); mInfo("acct:%s, will be created when deploying, raw:%p", acctObj.acct, pRaw);
@ -84,21 +88,24 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-acct");
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("acct:%s, failed to create since %s", acctObj.acct, terrstr()); code = terrno;
return -1; mError("acct:%s, failed to create since %s", acctObj.acct, tstrerror(code));
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct); mInfo("trans:%d, used to create acct:%s", pTrans->id, acctObj.acct);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { code = mndTransAppendCommitlog(pTrans, pRaw);
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); if (code != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { code = mndTransPrepare(pMnode, pTrans);
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); if (code != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, tstrerror(code));
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -216,31 +223,36 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
} }
static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { int32_t code = 0;
return -1; code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT);
if (code != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { int32_t code = 0;
return -1; code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT);
if (code != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) {
if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { int32_t code = 0;
return -1; if ((code = mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT)) != 0) {
TAOS_RETURN(code);
} }
terrno = TSDB_CODE_OPS_NOT_SUPPORT; code = TSDB_CODE_OPS_NOT_SUPPORT;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }

197
source/dnode/mnode/impl/src/mndArbGroup.c
View File

@ -57,6 +57,7 @@ static int32_t mndArbCheckToken(const char *token1, const char *token2) {
} }
int32_t mndInitArbGroup(SMnode *pMnode) { int32_t mndInitArbGroup(SMnode *pMnode) {
int32_t code = 0;
SSdbTable table = { SSdbTable table = {
.sdbType = SDB_ARBGROUP, .sdbType = SDB_ARBGROUP,
.keyType = SDB_KEY_INT32, .keyType = SDB_KEY_INT32,
@ -78,6 +79,10 @@ int32_t mndInitArbGroup(SMnode *pMnode) {
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_ARBGROUP, mndCancelGetNextArbGroup);
arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); arbUpdateHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
if (arbUpdateHash == NULL) {
code = terrno;
TAOS_RETURN(code);
}
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
@ -248,50 +253,74 @@ _OVER:
} }
int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupUndoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pUndoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pUndoRaw = mndArbGroupActionEncode(pGroup);
if (pUndoRaw == NULL) return -1; if (pUndoRaw == NULL) {
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendUndolog(pTrans, pUndoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetCreateArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw) != 0)) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetDropArbGroupPrepareLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
static int32_t mndSetDropArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) { static int32_t mndSetDropArbGroupRedoLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pRedoRaw = mndArbGroupActionEncode(pGroup);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) {
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendRedolog(pTrans, pRedoRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) { int32_t mndSetDropArbGroupCommitLogs(STrans *pTrans, SArbGroup *pGroup) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup); SSdbRaw *pCommitRaw = mndArbGroupActionEncode(pGroup);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = terrno;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) TAOS_RETURN(code);
if ((code = sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED)) != 0) TAOS_RETURN(code);
return 0; return 0;
} }
@ -343,6 +372,7 @@ static int32_t mndSendArbHeartBeatReq(SDnodeObj *pDnode, char *arbToken, int64_t
} }
static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) { static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL; SArbGroup *pArbGroup = NULL;
@ -377,7 +407,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
} }
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb timer"); mError("failed to get arb token for arb-hb timer");
pIter = taosHashIterate(pDnodeHash, NULL); pIter = taosHashIterate(pDnodeHash, NULL);
while (pIter) { while (pIter) {
@ -386,7 +416,7 @@ static int32_t mndProcessArbHbTimer(SRpcMsg *pReq) {
pIter = taosHashIterate(pDnodeHash, pIter); pIter = taosHashIterate(pDnodeHash, pIter);
} }
taosHashCleanup(pDnodeHash); taosHashCleanup(pDnodeHash);
return -1; TAOS_RETURN(code);
} }
int64_t nowMs = taosGetTimestampMs(); int64_t nowMs = taosGetTimestampMs();
@ -447,6 +477,7 @@ static void *mndBuildArbCheckSyncReq(int32_t *pContLen, int32_t vgId, char *arbT
static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbToken, int64_t term, char *member0Token, static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbToken, int64_t term, char *member0Token,
char *member1Token) { char *member1Token) {
int32_t code = 0;
int32_t contLen = 0; int32_t contLen = 0;
void *pHead = mndBuildArbCheckSyncReq(&contLen, vgId, arbToken, term, member0Token, member1Token); void *pHead = mndBuildArbCheckSyncReq(&contLen, vgId, arbToken, term, member0Token, member1Token);
if (!pHead) { if (!pHead) {
@ -459,10 +490,12 @@ static int32_t mndSendArbCheckSyncReq(SMnode *pMnode, int32_t vgId, char *arbTok
if (epSet.numOfEps == 0) { if (epSet.numOfEps == 0) {
mError("vgId:%d, failed to send check-sync request since no epSet found", vgId); mError("vgId:%d, failed to send check-sync request since no epSet found", vgId);
rpcFreeCont(pHead); rpcFreeCont(pHead);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = tmsgSendReq(&epSet, &rpcMsg); code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
mError("vgId:%d, failed to send check-sync request since 0x%x", vgId, code); mError("vgId:%d, failed to send check-sync request since 0x%x", vgId, code);
} else { } else {
@ -502,11 +535,14 @@ static void *mndBuildArbSetAssignedLeaderReq(int32_t *pContLen, int32_t vgId, ch
static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken, static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, int32_t vgId, char *arbToken,
int64_t term, char *memberToken) { int64_t term, char *memberToken) {
int32_t code = 0;
int32_t contLen = 0; int32_t contLen = 0;
void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken); void *pHead = mndBuildArbSetAssignedLeaderReq(&contLen, vgId, arbToken, term, memberToken);
if (!pHead) { if (!pHead) {
mError("vgId:%d, failed to build set-assigned request", vgId); mError("vgId:%d, failed to build set-assigned request", vgId);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
SRpcMsg rpcMsg = {.msgType = TDMT_SYNC_SET_ASSIGNED_LEADER, .pCont = pHead, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_SYNC_SET_ASSIGNED_LEADER, .pCont = pHead, .contLen = contLen};
@ -514,9 +550,11 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
if (epSet.numOfEps == 0) { if (epSet.numOfEps == 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since no epSet found", dnodeId, vgId); mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since no epSet found", dnodeId, vgId);
rpcFreeCont(pHead); rpcFreeCont(pHead);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int32_t code = tmsgSendReq(&epSet, &rpcMsg); code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since 0x%x", dnodeId, vgId, code); mError("dnodeId:%d vgId:%d, failed to send arb-set-assigned request to dnode since 0x%x", dnodeId, vgId, code);
} else { } else {
@ -526,6 +564,7 @@ static int32_t mndSendArbSetAssignedLeaderReq(SMnode *pMnode, int32_t dnodeId, i
} }
static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArbGroup *pArbGroup = NULL; SArbGroup *pArbGroup = NULL;
@ -533,14 +572,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) {
void *pIter = NULL; void *pIter = NULL;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync timer"); mError("failed to get arb token for arb-check-sync timer");
return -1; TAOS_RETURN(code);
} }
int64_t term = mndGetTerm(pMnode); int64_t term = mndGetTerm(pMnode);
if (term < 0) { if (term < 0) {
mError("arb failed to get term since %s", terrstr()); mError("arb failed to get term since %s", terrstr());
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
int64_t roleTimeMs = mndGetRoleTimeMs(pMnode); int64_t roleTimeMs = mndGetRoleTimeMs(pMnode);
@ -747,13 +788,13 @@ _OVER:
} }
static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) { static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
int ret = -1; int code = -1;
size_t sz = 0; size_t sz = 0;
SMArbUpdateGroupBatchReq req = {0}; SMArbUpdateGroupBatchReq req = {0};
if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) { if ((code = tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req)) != 0) {
mError("arb failed to decode arb-update-group request"); mError("arb failed to decode arb-update-group request");
return -1; TAOS_RETURN(code);
} }
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -789,7 +830,7 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
mndTransAddArbGroupId(pTrans, newGroup.vgId); mndTransAddArbGroupId(pTrans, newGroup.vgId);
if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) { if ((code = mndSetCreateArbGroupCommitLogs(pTrans, &newGroup)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id, mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id,
terrstr()); terrstr());
goto _OVER; goto _OVER;
@ -803,13 +844,13 @@ static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) {
sdbRelease(pMnode->pSdb, pOldGroup); sdbRelease(pMnode->pSdb, pOldGroup);
} }
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0; code = 0;
_OVER: _OVER:
if (ret != 0) { if (code != 0) {
// failed to update arbgroup // failed to update arbgroup
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i);
@ -819,7 +860,7 @@ _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
tFreeSMArbUpdateGroupBatchReq(&req); tFreeSMArbUpdateGroupBatchReq(&req);
return ret; return code;
} }
static void mndArbGroupDupObj(SArbGroup *pGroup, SArbGroup *pNew) { static void mndArbGroupDupObj(SArbGroup *pGroup, SArbGroup *pNew) {
@ -841,10 +882,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) {
} }
static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
int32_t ret = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to update arbgroup in create trans, vgId:%d, since %s", pNew->vgId, terrstr()); mError("failed to update arbgroup in create trans, vgId:%d, since %s", pNew->vgId, terrstr());
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -854,23 +896,22 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) {
pNew->assignedLeader.token, pNew->assignedLeader.acked); pNew->assignedLeader.token, pNew->assignedLeader.acked);
mndTransAddArbGroupId(pTrans, pNew->vgId); mndTransAddArbGroupId(pTrans, pNew->vgId);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if ((code = mndTransCheckConflict(pMnode, pTrans)) != 0) {
ret = -1;
goto _OVER; goto _OVER;
} }
if (mndSetCreateArbGroupCommitLogs(pTrans, pNew) != 0) { if ((code = mndSetCreateArbGroupCommitLogs(pTrans, pNew)) != 0) {
mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, terrstr()); mError("failed to update arbgroup in set commit log, vgId:%d, since %s", pNew->vgId, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; if ((code = mndTransPrepare(pMnode, pTrans)) != 0) goto _OVER;
ret = 0; code = 0;
_OVER: _OVER:
mndTransDrop(pTrans); mndTransDrop(pTrans);
return ret; return code;
} }
bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId, bool mndUpdateArbGroupByHeartBeat(SArbGroup *pGroup, SVArbHbRspMember *pRspMember, int64_t nowMs, int32_t dnodeId,
@ -993,11 +1034,13 @@ _OVER:
} }
static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token, char *member1Token, bool newIsSync) { static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token, char *member1Token, bool newIsSync) {
int32_t code = 0;
SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId); SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId);
if (pGroup == NULL) { if (pGroup == NULL) {
terrno = TSDB_CODE_NOT_FOUND;
mInfo("failed to update arb sync, vgId:%d not found", vgId); mInfo("failed to update arb sync, vgId:%d not found", vgId);
return -1; code = -1;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
SArbGroup newGroup = {0}; SArbGroup newGroup = {0};
@ -1018,38 +1061,36 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-hb response"); mError("failed to get arb token for arb-hb response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbHeartBeatRsp arbHbRsp = {0}; SVArbHeartBeatRsp arbHbRsp = {0};
if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) { if ((code = tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp)) != 0) {
mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, arbHbRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, arbHbRsp.arbToken) != 0) {
mInfo("arb hearbeat skip update for dnodeId:%d, arb token mismatch, local:[%s] msg:[%s]", arbHbRsp.dnodeId, mInfo("arb hearbeat skip update for dnodeId:%d, arb token mismatch, local:[%s] msg:[%s]", arbHbRsp.dnodeId,
arbToken, arbHbRsp.arbToken); arbToken, arbHbRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH; code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER; goto _OVER;
} }
(void)mndUpdateArbHeartBeat(pMnode, arbHbRsp.dnodeId, arbHbRsp.hbMembers); (void)mndUpdateArbHeartBeat(pMnode, arbHbRsp.dnodeId, arbHbRsp.hbMembers);
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbHeartBeatRsp(&arbHbRsp); tFreeSVArbHeartBeatRsp(&arbHbRsp);
return ret; return code;
} }
static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
@ -1058,27 +1099,25 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-check-sync response"); mError("failed to get arb token for arb-check-sync response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbCheckSyncRsp syncRsp = {0}; SVArbCheckSyncRsp syncRsp = {0};
if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) { if ((code = tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp)) != 0) {
mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code));
if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) { if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) {
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
return 0; return 0;
} }
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, syncRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, syncRsp.arbToken) != 0) {
@ -1089,16 +1128,16 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) {
} }
bool newIsSync = (syncRsp.errCode == TSDB_CODE_SUCCESS); bool newIsSync = (syncRsp.errCode == TSDB_CODE_SUCCESS);
if (mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync) != 0) { if ((code = mndUpdateArbSync(pMnode, syncRsp.vgId, syncRsp.member0Token, syncRsp.member1Token, newIsSync)) != 0) {
mInfo("failed to update arb sync for vgId:%d, since:%s", syncRsp.vgId, terrstr()); mInfo("failed to update arb sync for vgId:%d, since:%s", syncRsp.vgId, terrstr());
goto _OVER; goto _OVER;
} }
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbCheckSyncRsp(&syncRsp); tFreeSVArbCheckSyncRsp(&syncRsp);
return ret; TAOS_RETURN(code);
} }
bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode, bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char *memberToken, int32_t errcode,
@ -1138,35 +1177,35 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
return 0; return 0;
} }
int32_t ret = -1; int32_t code = -1;
SMnode *pMnode = pRsp->info.node; SMnode *pMnode = pRsp->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
char arbToken[TSDB_ARB_TOKEN_SIZE]; char arbToken[TSDB_ARB_TOKEN_SIZE];
if (mndGetArbToken(pMnode, arbToken) != 0) { if ((code = mndGetArbToken(pMnode, arbToken)) != 0) {
mError("failed to get arb token for arb-set-assigned response"); mError("failed to get arb token for arb-set-assigned response");
terrno = TSDB_CODE_NOT_FOUND; TAOS_RETURN(code);
return -1;
} }
SVArbSetAssignedLeaderRsp setAssignedRsp = {0}; SVArbSetAssignedLeaderRsp setAssignedRsp = {0};
if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) { if ((code = tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp)) != 0) {
mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code)); mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code));
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(code);
return -1;
} }
if (mndArbCheckToken(arbToken, setAssignedRsp.arbToken) != 0) { if (mndArbCheckToken(arbToken, setAssignedRsp.arbToken) != 0) {
mInfo("skip update arb assigned for vgId:%d, arb token mismatch, local:[%s] msg:[%s]", setAssignedRsp.vgId, mInfo("skip update arb assigned for vgId:%d, arb token mismatch, local:[%s] msg:[%s]", setAssignedRsp.vgId,
arbToken, setAssignedRsp.arbToken); arbToken, setAssignedRsp.arbToken);
terrno = TSDB_CODE_MND_ARB_TOKEN_MISMATCH; code = TSDB_CODE_MND_ARB_TOKEN_MISMATCH;
goto _OVER; goto _OVER;
} }
SArbGroup *pGroup = mndAcquireArbGroup(pMnode, setAssignedRsp.vgId); SArbGroup *pGroup = mndAcquireArbGroup(pMnode, setAssignedRsp.vgId);
if (!pGroup) { if (!pGroup) {
mError("failed to set arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr()); mError("failed to set arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr());
code = -1;
if (terrno != 0) code = terrno;
goto _OVER; goto _OVER;
} }
@ -1174,17 +1213,17 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) {
bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(pGroup, setAssignedRsp.vgId, setAssignedRsp.memberToken, bool updateAssigned = mndUpdateArbGroupBySetAssignedLeader(pGroup, setAssignedRsp.vgId, setAssignedRsp.memberToken,
pRsp->code, &newGroup); pRsp->code, &newGroup);
if (updateAssigned) { if (updateAssigned) {
if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { if ((code = mndPullupArbUpdateGroup(pMnode, &newGroup)) != 0) {
mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, terrstr()); mInfo("failed to pullup update arb assigned for vgId:%d, since:%s", setAssignedRsp.vgId, tstrerror(code));
goto _OVER; goto _OVER;
} }
} }
ret = 0; code = 0;
_OVER: _OVER:
tFreeSVArbSetAssignedLeaderRsp(&setAssignedRsp); tFreeSVArbSetAssignedLeaderRsp(&setAssignedRsp);
return ret; return code;
} }
static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {

58
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -14,8 +14,8 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "audit.h"
#include "mndCluster.h" #include "mndCluster.h"
#include "audit.h"
#include "mndGrant.h" #include "mndGrant.h"
#include "mndPrivilege.h" #include "mndPrivilege.h"
#include "mndShow.h" #include "mndShow.h"
@ -61,11 +61,13 @@ int32_t mndInitCluster(SMnode *pMnode) {
void mndCleanupCluster(SMnode *pMnode) {} void mndCleanupCluster(SMnode *pMnode) {}
int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
SSdb *pSdb = pMnode->pSdb; int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId);
if (pCluster == NULL) { if (pCluster == NULL) {
return -1; code = terrno;
TAOS_RETURN(code);
} }
tstrncpy(clusterName, pCluster->name, len); tstrncpy(clusterName, pCluster->name, len);
@ -247,7 +249,10 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
mInfo("cluster:%" PRId64 ", name is %s", clusterObj.id, clusterObj.name); mInfo("cluster:%" PRId64 ", name is %s", clusterObj.id, clusterObj.name);
SSdbRaw *pRaw = mndClusterActionEncode(&clusterObj); SSdbRaw *pRaw = mndClusterActionEncode(&clusterObj);
if (pRaw == NULL) return -1; if (pRaw == NULL) {
code = terrno;
TAOS_RETURN(code);
}
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw); mInfo("cluster:%" PRId64 ", will be created when deploying, raw:%p", clusterObj.id, pRaw);
@ -256,21 +261,22 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) {
if (pTrans == NULL) { if (pTrans == NULL) {
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr());
return -1; code = terrno;
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id); mInfo("trans:%d, used to create cluster:%" PRId64, pTrans->id, clusterObj.id);
if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { if ((code = mndTransAppendCommitlog(pTrans, pRaw)) != 0) {
mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -347,22 +353,33 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
int32_t code = 0;
mInfo("update cluster uptime to %d", clusterObj.upTime); mInfo("update cluster uptime to %d", clusterObj.upTime);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "update-uptime");
if (pTrans == NULL) return -1; if (pTrans == NULL) {
code = terrno;
TAOS_RETURN(code);
}
SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -374,13 +391,12 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SMCfgClusterReq cfgReq = {0}; SMCfgClusterReq cfgReq = {0};
if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
return -1; TAOS_RETURN(code);
} }
mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER) != 0) { if ((code = mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER)) != 0) {
code = terrno != 0 ? terrno : TSDB_CODE_MND_NO_RIGHTS;
goto _exit; goto _exit;
} }
@ -410,17 +426,17 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
} }
{ // audit { // audit
auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql,
TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1));
} }
_exit: _exit:
tFreeSMCfgClusterReq(&cfgReq); tFreeSMCfgClusterReq(&cfgReq);
if (code != 0) { if (code != 0) {
terrno = code;
mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr()); mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr());
} else { } else {
mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value); mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value);
} }
return code; TAOS_RETURN(code);
} }
int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) { int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) {

183
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -75,11 +75,11 @@ int32_t tDeserializeSCompactObj(void *buf, int32_t bufLen, SCompactObj *pObj) {
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(&decoder));
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
if (tDecodeCStrTo(&decoder, pObj->dbname) < 0) return -1; TAOS_CHECK_RETURN(tDecodeCStrTo(&decoder, pObj->dbname));
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
tEndDecode(&decoder); tEndDecode(&decoder);
@ -174,13 +174,10 @@ SSdbRow *mndCompactActionDecode(SSdbRaw *pRaw) {
} }
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER); SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactObj(buf, tlen, pCompact) < 0) { if ((terrno = tDeserializeSCompactObj(buf, tlen, pCompact)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER; goto OVER;
} }
// taosInitRWLatch(&pView->lock);
OVER: OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
@ -227,6 +224,7 @@ void mndReleaseCompact(SMnode *pMnode, SCompactObj *pCompact) {
// compact db // compact db
int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) { int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompact, SDbObj *pDb, SCompactDbRsp *rsp) {
int32_t code = 0;
pCompact->compactId = tGenIdPI32(); pCompact->compactId = tGenIdPI32();
strcpy(pCompact->dbname, pDb->name); strcpy(pCompact->dbname, pDb->name);
@ -234,10 +232,14 @@ int32_t mndAddCompactToTran(SMnode *pMnode, STrans *pTrans, SCompactObj *pCompac
pCompact->startTime = taosGetTimestampMs(); pCompact->startTime = taosGetTimestampMs();
SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact); SSdbRaw *pVgRaw = mndCompactActionEncode(pCompact);
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) {
if (mndTransAppendPrepareLog(pTrans, pVgRaw) != 0) { code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendPrepareLog(pTrans, pVgRaw)) != 0) {
sdbFreeRaw(pVgRaw); sdbFreeRaw(pVgRaw);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
@ -335,44 +337,62 @@ static void *mndBuildKillCompactReq(SMnode *pMnode, SVgObj *pVgroup, int32_t *pC
static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId, static int32_t mndAddKillCompactAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t compactId,
int32_t dnodeid) { int32_t dnodeid) {
int32_t code = 0;
STransAction action = {0}; STransAction action = {0};
SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid); SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeid);
if (pDnode == NULL) return -1; if (pDnode == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.epSet = mndGetDnodeEpset(pDnode); action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid); void *pReq = mndBuildKillCompactReq(pMnode, pVgroup, &contLen, compactId, dnodeid);
if (pReq == NULL) return -1; if (pReq == NULL) {
code = TSDB_CODE_SDB_OBJ_NOT_THERE;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
action.pCont = pReq; action.pCont = pReq;
action.contLen = contLen; action.contLen = contLen;
action.msgType = TDMT_VND_KILL_COMPACT; action.msgType = TDMT_VND_KILL_COMPACT;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if ((code = mndTransAppendRedoAction(pTrans, &action)) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
} }
static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) { static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompact) {
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "kill-compact");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr()); mError("compact:%" PRId32 ", failed to drop since %s", pCompact->compactId, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId); mInfo("trans:%d, used to kill compact:%" PRId32, pTrans->id, pCompact->compactId);
mndTransSetDbName(pTrans, pCompact->dbname, NULL); mndTransSetDbName(pTrans, pCompact->dbname, NULL);
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
mndTransDrop(pTrans);
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
@ -386,14 +406,20 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId); SVgObj *pVgroup = mndAcquireVgroup(pMnode, pDetail->vgId);
if (pVgroup == NULL) { if (pVgroup == NULL) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
if (mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId) != 0) { if ((code = mndAddKillCompactAction(pMnode, pTrans, pVgroup, pCompact->compactId, pDetail->dnodeId)) != 0) {
mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr()); mError("trans:%d, failed to append redo action since %s", pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndReleaseVgroup(pMnode, pVgroup); mndReleaseVgroup(pMnode, pVgroup);
@ -412,10 +438,10 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -423,30 +449,27 @@ static int32_t mndKillCompact(SMnode *pMnode, SRpcMsg *pReq, SCompactObj *pCompa
} }
int32_t mndProcessKillCompactReq(SRpcMsg *pReq) { int32_t mndProcessKillCompactReq(SRpcMsg *pReq) {
int32_t code = 0;
int32_t lino = 0;
SKillCompactReq killCompactReq = {0}; SKillCompactReq killCompactReq = {0};
if (tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; if ((code = tDeserializeSKillCompactReq(pReq->pCont, pReq->contLen, &killCompactReq)) != 0) {
return -1; TAOS_RETURN(code);
} }
mInfo("start to kill compact:%" PRId32, killCompactReq.compactId); mInfo("start to kill compact:%" PRId32, killCompactReq.compactId);
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId); SCompactObj *pCompact = mndAcquireCompact(pMnode, killCompactReq.compactId);
if (pCompact == NULL) { if (pCompact == NULL) {
terrno = TSDB_CODE_MND_INVALID_COMPACT_ID; code = TSDB_CODE_MND_INVALID_COMPACT_ID;
tFreeSKillCompactReq(&killCompactReq); tFreeSKillCompactReq(&killCompactReq);
return -1; TAOS_RETURN(code);
} }
if (0 != mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB)) { TAOS_CHECK_GOTO(mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB), &lino, _OVER);
goto _OVER;
}
if (mndKillCompact(pMnode, pReq, pCompact) < 0) { TAOS_CHECK_GOTO(mndKillCompact(pMnode, pReq, pCompact), &lino, _OVER);
goto _OVER;
}
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -463,12 +486,14 @@ _OVER:
tFreeSKillCompactReq(&killCompactReq); tFreeSKillCompactReq(&killCompactReq);
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
return code; TAOS_RETURN(code);
} }
// update progress // update progress
static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId, static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t compactId,
SQueryCompactProgressRsp *rsp) { SQueryCompactProgressRsp *rsp) {
int32_t code = 0;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
SCompactDetailObj *pDetail = NULL; SCompactDetailObj *pDetail = NULL;
@ -479,9 +504,10 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
pDetail->newNumberFileset = rsp->numberFileset; pDetail->newNumberFileset = rsp->numberFileset;
pDetail->newFinished = rsp->finished; pDetail->newFinished = rsp->finished;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
return 0; TAOS_RETURN(code);
} }
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
@ -491,14 +517,13 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
} }
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) { int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
SQueryCompactProgressRsp req = {0};
int32_t code = 0; int32_t code = 0;
SQueryCompactProgressRsp req = {0};
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_INVALID_MSG;
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont, mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,
pReq->contLen); pReq->contLen);
return -1; TAOS_RETURN(code);
} }
mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, mDebug("compact:%d, receive query response, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
@ -508,13 +533,12 @@ int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req); code = mndUpdateCompactProgress(pMnode, pReq, req.compactId, &req);
if (code != 0) { if (code != 0) {
terrno = code;
mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId, mError("compact:%d, failed to update progress, vgId:%d, dnodeId:%d, numberFileset:%d, finished:%d", req.compactId,
req.vgId, req.dnodeId, req.numberFileset, req.finished); req.vgId, req.dnodeId, req.numberFileset, req.finished);
return -1; TAOS_RETURN(code);
} }
return 0; TAOS_RETURN(code);
} }
// timer // timer
@ -531,7 +555,10 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId); SDnodeObj *pDnode = mndAcquireDnode(pMnode, pDetail->dnodeId);
if (pDnode == NULL) break; if (pDnode == NULL) break;
addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port); if (addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port) != 0) {
sdbRelease(pMnode->pSdb, pDetail);
continue;
}
mndReleaseDnode(pMnode, pDnode); mndReleaseDnode(pMnode, pDnode);
SQueryCompactProgressReq req; SQueryCompactProgressReq req;
@ -541,8 +568,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req); int32_t contLen = tSerializeSQueryCompactProgressReq(NULL, 0, &req);
if (contLen < 0) { if (contLen < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
continue; continue;
} }
@ -551,7 +576,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SMsgHead *pHead = rpcMallocCont(contLen); SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) { if (pHead == NULL) {
sdbCancelFetch(pMnode->pSdb, pDetail);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
continue; continue;
} }
@ -563,13 +587,6 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_VND_QUERY_COMPACT_PROGRESS, .contLen = contLen};
// rpcMsg.pCont = rpcMallocCont(contLen);
// if (rpcMsg.pCont == NULL) {
// return;
// }
// memcpy(rpcMsg.pCont, pHead, contLen);
rpcMsg.pCont = pHead; rpcMsg.pCont = pHead;
char detail[1024] = {0}; char detail[1024] = {0};
@ -589,6 +606,7 @@ void mndCompactSendProgressReq(SMnode *pMnode, SCompactObj *pCompact) {
} }
static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) { static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
int32_t code = 0;
bool needSave = false; bool needSave = false;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
@ -612,7 +630,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId); SCompactObj *pCompact = mndAcquireCompact(pMnode, compactId);
if (pCompact == NULL) return 0; if (pCompact == NULL) TAOS_RETURN(code);
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname); SDbObj *pDb = mndAcquireDb(pMnode, pCompact->dbname);
if (pDb == NULL) { if (pDb == NULL) {
@ -625,13 +643,15 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (!needSave) { if (!needSave) {
mDebug("compact:%" PRId32 ", no need to save", compactId); mDebug("compact:%" PRId32 ", no need to save", compactId);
return 0; TAOS_RETURN(code);
} }
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, NULL, "update-compact-progress");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr()); mError("trans:%" PRId32 ", failed to create since %s", pTrans->id, terrstr());
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id); mInfo("compact:%d, trans:%d, used to update compact progress.", compactId, pTrans->id);
@ -654,10 +674,20 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
pDetail->finished = pDetail->newFinished; pDetail->finished = pDetail->newFinished;
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail); SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr()); sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans);
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
} }
@ -678,11 +708,13 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->numberFileset == -1 && pDetail->finished == -1) { if (pDetail->numberFileset == -1 && pDetail->finished == -1) {
allFinished = false; allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
break; break;
} }
if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) { if (pDetail->numberFileset != -1 && pDetail->finished != -1 && pDetail->numberFileset != pDetail->finished) {
allFinished = false; allFinished = false;
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail); sdbRelease(pMnode->pSdb, pDetail);
break; break;
} }
@ -710,11 +742,19 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
if (pDetail->compactId == pCompact->compactId) { if (pDetail->compactId == pCompact->compactId) {
SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail); SSdbRaw *pCommitRaw = mndCompactDetailActionEncode(pDetail);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id, mError("compact:%d, trans:%d, failed to append commit log since %s", pDetail->compactId, pTrans->id,
terrstr()); terrstr());
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pDetail);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId); mInfo("compact:%d, add drop compactdetail action", pDetail->compactDetailId);
@ -724,20 +764,26 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact); SSdbRaw *pCommitRaw = mndCompactActionEncode(pCompact);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) {
mndTransDrop(pTrans);
code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
}
if ((code = mndTransAppendCommitlog(pTrans, pCommitRaw)) != 0) {
mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr()); mError("compact:%d, trans:%d, failed to append commit log since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
mInfo("compact:%d, add drop compact action", pCompact->compactId); mInfo("compact:%d, add drop compact action", pCompact->compactId);
} }
if (mndTransPrepare(pMnode, pTrans) != 0) { if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr()); mError("compact:%d, trans:%d, failed to prepare since %s", compactId, pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
return -1; TAOS_RETURN(code);
} }
sdbRelease(pMnode->pSdb, pCompact); sdbRelease(pMnode->pSdb, pCompact);
@ -746,6 +792,7 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
} }
void mndCompactPullup(SMnode *pMnode) { void mndCompactPullup(SMnode *pMnode) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t)); SArray *pArray = taosArrayInit(sdbGetSize(pSdb, SDB_COMPACT), sizeof(int32_t));
if (pArray == NULL) return; if (pArray == NULL) return;
@ -766,7 +813,9 @@ void mndCompactPullup(SMnode *pMnode) {
if (pCompact != NULL) { if (pCompact != NULL) {
mInfo("compact:%d, begin to pull up", pCompact->compactId); mInfo("compact:%d, begin to pull up", pCompact->compactId);
mndCompactSendProgressReq(pMnode, pCompact); mndCompactSendProgressReq(pMnode, pCompact);
mndSaveCompactProgress(pMnode, pCompact->compactId); if ((code = mndSaveCompactProgress(pMnode, pCompact->compactId)) != 0) {
mError("compact:%d, failed to save compact progress since %s", pCompact->compactId, tstrerror(code));
}
} }
mndReleaseCompact(pMnode, pCompact); mndReleaseCompact(pMnode, pCompact);
} }

34
source/dnode/mnode/impl/src/mndCompactDetail.c
View File

@ -125,17 +125,17 @@ int32_t tDeserializeSCompactDetailObj(void *buf, int32_t bufLen, SCompactDetailO
SDecoder decoder = {0}; SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen); tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1; TAOS_CHECK_RETURN(tStartDecode(&decoder));
if (tDecodeI32(&decoder, &pObj->compactDetailId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactDetailId));
if (tDecodeI32(&decoder, &pObj->compactId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->compactId));
if (tDecodeI32(&decoder, &pObj->vgId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->vgId));
if (tDecodeI32(&decoder, &pObj->dnodeId) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->dnodeId));
if (tDecodeI32(&decoder, &pObj->numberFileset) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->numberFileset));
if (tDecodeI32(&decoder, &pObj->finished) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->finished));
if (tDecodeI64(&decoder, &pObj->startTime) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI64(&decoder, &pObj->startTime));
if (tDecodeI32(&decoder, &pObj->newNumberFileset) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newNumberFileset));
if (tDecodeI32(&decoder, &pObj->newFinished) < 0) return -1; TAOS_CHECK_RETURN(tDecodeI32(&decoder, &pObj->newFinished));
tEndDecode(&decoder); tEndDecode(&decoder);
@ -231,13 +231,10 @@ SSdbRow *mndCompactDetailActionDecode(SSdbRaw *pRaw) {
} }
SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER); SDB_GET_BINARY(pRaw, dataPos, buf, tlen, OVER);
if (tDeserializeSCompactDetailObj(buf, tlen, pCompact) < 0) { if ((terrno = tDeserializeSCompactDetailObj(buf, tlen, pCompact)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto OVER; goto OVER;
} }
//taosInitRWLatch(&pView->lock);
OVER: OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
@ -274,6 +271,7 @@ int32_t mndCompactDetailActionUpdate(SSdb *pSdb, SCompactDetailObj *pOldCompact,
int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup, int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* pCompact, SVgObj *pVgroup,
SVnodeGid *pVgid, int32_t index){ SVnodeGid *pVgid, int32_t index){
int32_t code = 0;
SCompactDetailObj compactDetail = {0}; SCompactDetailObj compactDetail = {0};
compactDetail.compactDetailId = index; compactDetail.compactDetailId = index;
compactDetail.compactId = pCompact->compactId; compactDetail.compactId = pCompact->compactId;
@ -292,9 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p
if (pVgRaw == NULL) return -1; if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) {
sdbFreeRaw(pVgRaw); sdbFreeRaw(pVgRaw);
return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
TAOS_RETURN(code);
} }
(void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
return 0; TAOS_RETURN(code);
} }

129
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -95,26 +95,21 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
bool enableReplay) { bool enableReplay) {
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
int32_t numOfTopics = taosArrayGetSize(pTopicList); int32_t numOfTopics = taosArrayGetSize(pTopicList);
for (int32_t i = 0; i < numOfTopics; i++) { for (int32_t i = 0; i < numOfTopics; i++) {
char *pOneTopic = taosArrayGetP(pTopicList, i); char *pOneTopic = taosArrayGetP(pTopicList, i);
pTopic = mndAcquireTopic(pMnode, pOneTopic); pTopic = mndAcquireTopic(pMnode, pOneTopic);
if (pTopic == NULL) { // terrno has been set by callee function if (pTopic == NULL) { // terrno has been set by callee function
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED; goto FAILED;
} }
if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) { TAOS_CHECK_GOTO(mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic), &lino, FAILED);
code = TSDB_CODE_MND_NO_RIGHTS;
terrno = TSDB_CODE_MND_NO_RIGHTS;
goto FAILED;
}
if ((terrno = grantCheckExpire(TSDB_GRANT_SUBSCRIPTION)) < 0) { TAOS_CHECK_GOTO(grantCheckExpire(TSDB_GRANT_SUBSCRIPTION), &lino, FAILED);
code = terrno;
goto FAILED;
}
if (enableReplay) { if (enableReplay) {
if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) {
@ -123,7 +118,8 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
} else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { } else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
if (pDb == NULL) { if (pDb == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto FAILED; goto FAILED;
} }
if (pDb->cfg.numOfVgroups != 1) { if (pDb->cfg.numOfVgroups != 1) {
@ -141,11 +137,12 @@ static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const ch
return 0; return 0;
FAILED: FAILED:
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
@ -153,7 +150,8 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId); mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
@ -161,42 +159,39 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
pConsumer->status, mndConsumerStatusName(pConsumer->status)); pConsumer->status, mndConsumerStatusName(pConsumer->status));
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) { if (pConsumer->status != MQ_CONSUMER_STATUS_LOST) {
terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; code = TSDB_CODE_MND_CONSUMER_NOT_READY;
code = -1;
goto END; goto END;
} }
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL); pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, CONSUMER_UPDATE_REC, NULL, NULL);
if (pConsumerNew == NULL){ if (pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) { if (pTrans == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
goto END; if (terrno != 0) code = terrno;
}
code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) {
goto END; goto END;
} }
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew); TAOS_CHECK_GOTO(validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false), &lino, END);
if (code != 0) {
goto END; TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), &lino, END);
}
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
END: END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqConsumerClearMsg *pClearMsg = pMsg->pCont; SMqConsumerClearMsg *pClearMsg = pMsg->pCont;
SMqConsumerObj *pConsumerNew = NULL; SMqConsumerObj *pConsumerNew = NULL;
@ -205,7 +200,7 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId); mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId);
return 0; TAOS_RETURN(code);
} }
mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId,
@ -213,21 +208,20 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) {
pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL); pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup, -1, NULL, NULL);
if (pConsumerNew == NULL){ if (pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm");
if (pTrans == NULL) { if (pTrans == NULL) {
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto END; goto END;
} }
// this is the drop action, not the update action // this is the drop action, not the update action
code = mndSetConsumerDropLogs(pTrans, pConsumerNew); TAOS_CHECK_GOTO(mndSetConsumerDropLogs(pTrans, pConsumerNew), &lino, END);
if (code != 0) {
goto END;
}
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
@ -235,7 +229,7 @@ END:
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; TAOS_RETURN(code);
} }
static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) {
@ -310,10 +304,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqHbRsp rsp = {0}; SMqHbRsp rsp = {0};
SMqConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) { TAOS_CHECK_GOTO(tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req), NULL, end);
code = TSDB_CODE_TMQ_INVALID_MSG;
goto end;
}
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
pConsumer = mndAcquireConsumer(pMnode, consumerId); pConsumer = mndAcquireConsumer(pMnode, consumerId);
@ -322,10 +313,8 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
code = TSDB_CODE_MND_CONSUMER_NOT_EXIST; code = TSDB_CODE_MND_CONSUMER_NOT_EXIST;
goto end; goto end;
} }
code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user);
if (code != 0) { TAOS_CHECK_GOTO(checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user), NULL, end);
goto end;
}
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
@ -343,7 +332,7 @@ end:
tDestroySMqHbRsp(&rsp); tDestroySMqHbRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
tDestroySMqHbReq(&req); tDestroySMqHbReq(&req);
return code; TAOS_RETURN(code);
} }
static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t epoch, SMqAskEpRsp *rsp){
@ -431,6 +420,7 @@ static int32_t addEpSetInfo(SMnode *pMnode, SMqConsumerObj *pConsumer, int32_t e
} }
static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoch, int64_t consumerId){
int32_t code = 0;
// encode rsp // encode rsp
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp); int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, rsp);
void *buf = rpcMallocCont(tlen); void *buf = rpcMallocCont(tlen);
@ -452,7 +442,7 @@ static int32_t buildAskEpRsp(SRpcMsg *pMsg, SMqAskEpRsp *rsp, int32_t serverEpoc
// send rsp // send rsp
pMsg->info.rsp = buf; pMsg->info.rsp = buf;
pMsg->info.rspLen = tlen; pMsg->info.rspLen = tlen;
return 0; TAOS_RETURN(code);
} }
static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
@ -461,9 +451,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
SMqAskEpRsp rsp = {0}; SMqAskEpRsp rsp = {0};
int32_t code = 0; int32_t code = 0;
if (tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req) < 0) { TAOS_CHECK_RETURN(tDeserializeSMqAskEpReq(pMsg->pCont, pMsg->contLen, &req));
return TSDB_CODE_OUT_OF_MEMORY;
}
int64_t consumerId = req.consumerId; int64_t consumerId = req.consumerId;
@ -517,19 +505,29 @@ END:
} }
int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerDropLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED));
TAOS_RETURN(code);
} }
int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) { int32_t mndSetConsumerCommitLogs(STrans *pTrans, SMqConsumerObj *pConsumer) {
int32_t code = 0;
SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer); SSdbRaw *pCommitRaw = mndConsumerActionEncode(pConsumer);
if (pCommitRaw == NULL) return -1; if (pCommitRaw == NULL) {
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; if (terrno != 0) code = terrno;
return 0; TAOS_RETURN(code);
}
TAOS_CHECK_RETURN(mndTransAppendCommitlog(pTrans, pCommitRaw));
TAOS_CHECK_RETURN(sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY));
TAOS_RETURN(code);
} }
static void freeItem(void *param) { static void freeItem(void *param) {
@ -585,6 +583,7 @@ static int32_t getTopicAddDelete(SMqConsumerObj *pExistedConsumer, SMqConsumerOb
} }
static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
int32_t code = 0;
taosArraySort(pTopicList, taosArrayCompareString); taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);
@ -592,11 +591,10 @@ static int32_t checkAndSortTopic(SMnode *pMnode, SArray *pTopicList){
for (int i = 0; i < newTopicNum; i++) { for (int i = 0; i < newTopicNum; i++) {
int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i)); int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i));
if (gNum >= MND_MAX_GROUP_PER_TOPIC) { if (gNum >= MND_MAX_GROUP_PER_TOPIC) {
terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; return TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE;
return -1;
} }
} }
return TSDB_CODE_SUCCESS; TAOS_RETURN(code);
} }
static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){ static SMqConsumerObj* buildSubConsumer(SMnode *pMnode, SCMSubscribeReq *subscribe){
@ -672,7 +670,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over; goto _over;
} }
@ -683,21 +682,21 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
pConsumerNew = buildSubConsumer(pMnode, &subscribe); pConsumerNew = buildSubConsumer(pMnode, &subscribe);
if(pConsumerNew == NULL){ if(pConsumerNew == NULL){
code = -1; code = TSDB_CODE_MND_RETURN_VALUE_NULL;
if (terrno != 0) code = terrno;
goto _over; goto _over;
} }
code = mndSetConsumerCommitLogs(pTrans, pConsumerNew);
if (code != 0) goto _over;
code = mndTransPrepare(pMnode, pTrans); TAOS_CHECK_GOTO(mndSetConsumerCommitLogs(pTrans, pConsumerNew), NULL, _over);
if (code != 0) goto _over;
TAOS_CHECK_GOTO(mndTransPrepare(pMnode, pTrans), NULL, _over);
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
_over: _over:
mndTransDrop(pTrans); mndTransDrop(pTrans);
tDeleteSMqConsumerObj(pConsumerNew); tDeleteSMqConsumerObj(pConsumerNew);
taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree); taosArrayDestroyP(subscribe.topicNames, (FDelete)taosMemoryFree);
return code; TAOS_RETURN(code);
} }
SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {

1
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -105,6 +105,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) {
mInfo("vgId:%d replica:%d inconsistent with other vgroups replica:%d, not ready for stream operations", mInfo("vgId:%d replica:%d inconsistent with other vgroups replica:%d, not ready for stream operations",
pVgroup->vgId, pVgroup->replica, replica); pVgroup->vgId, pVgroup->replica, replica);
*allReady = false; *allReady = false;
sdbRelease(pSdb, pVgroup);
break; break;
} }
} }

54
source/dnode/mnode/impl/test/sdb/sdbTest.cpp
View File

@ -387,11 +387,11 @@ TEST_F(MndTestSdb, 00_API) {
void *pRow2 = sdbGetRowObj(NULL); void *pRow2 = sdbGetRowObj(NULL);
ASSERT_EQ(pRow2 == NULL, 1); ASSERT_EQ(pRow2 == NULL, 1);
//sdbRaw.c // sdbRaw.c
SStrObj strObj; SStrObj strObj;
SSdbRaw *pRaw1 = NULL; SSdbRaw *pRaw1 = NULL;
strSetDefault(&strObj, 1); strSetDefault(&strObj, 1);
pRaw1 = strEncode(&strObj); pRaw1 = strEncode(&strObj);
int32_t id = sdbGetIdFromRaw(pSdb, pRaw1); int32_t id = sdbGetIdFromRaw(pSdb, pRaw1);
ASSERT_EQ(id, -2); ASSERT_EQ(id, -2);
@ -399,32 +399,32 @@ TEST_F(MndTestSdb, 00_API) {
SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128); SSdbRaw *pRaw2 = sdbAllocRaw(SDB_USER, 1, -128);
ASSERT_EQ(pRaw2 == NULL, 1); ASSERT_EQ(pRaw2 == NULL, 1);
ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt8(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt32(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt16(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), -1); ASSERT_EQ(sdbSetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), -1); ASSERT_EQ(sdbSetRawInt64(pRaw1, -128, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), -1); ASSERT_EQ(sdbSetRawBinary(NULL, 0, "12", 3), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), -1); ASSERT_EQ(sdbSetRawBinary(pRaw1, 9028, "12", 3), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawDataLen(NULL, 0), -1); ASSERT_EQ(sdbSetRawDataLen(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), -1); ASSERT_EQ(sdbSetRawDataLen(pRaw1, 9000), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), -1); ASSERT_EQ(sdbSetRawStatus(NULL, SDB_STATUS_READY), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), -1); ASSERT_EQ(sdbSetRawStatus(pRaw1, SDB_STATUS_INIT), TSDB_CODE_INVALID_PARA);
ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt8(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt8(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt32(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt32(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt16(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt16(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), -1); ASSERT_EQ(sdbGetRawInt64(NULL, 0, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), -1); ASSERT_EQ(sdbGetRawInt64(pRaw1, 9000, 0), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), -1); ASSERT_EQ(sdbGetRawBinary(NULL, 0, 0, 4096), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), -1); ASSERT_EQ(sdbGetRawBinary(pRaw1, 9000, 0, 112), TSDB_CODE_SDB_INVALID_DATA_LEN);
ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), -1); ASSERT_EQ(sdbGetRawSoftVer(NULL, 0), TSDB_CODE_INVALID_PTR);
ASSERT_EQ(sdbGetRawTotalSize(NULL), -1); ASSERT_EQ(sdbGetRawTotalSize(NULL), -1);
// sdbHash.c // sdbHash.c

19
source/dnode/mnode/sdb/src/sdb.c
View File

@ -112,6 +112,8 @@ void sdbCleanup(SSdb *pSdb) {
} }
int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) { int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
int32_t code = 0;
ESdbType sdbType = table.sdbType; ESdbType sdbType = table.sdbType;
EKeyType keyType = table.keyType; EKeyType keyType = table.keyType;
pSdb->keyTypes[sdbType] = table.keyType; pSdb->keyTypes[sdbType] = table.keyType;
@ -134,8 +136,8 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_ENTRY_LOCK); SHashObj *hash = taosHashInit(64, taosGetDefaultHashFunction(hashType), true, HASH_ENTRY_LOCK);
if (hash == NULL) { if (hash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
pSdb->maxId[sdbType] = 0; pSdb->maxId[sdbType] = 0;
@ -146,16 +148,17 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
} }
static int32_t sdbCreateDir(SSdb *pSdb) { static int32_t sdbCreateDir(SSdb *pSdb) {
int32_t code = 0;
if (taosMulMkDir(pSdb->currDir) != 0) { if (taosMulMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr()); mError("failed to create dir:%s since %s", pSdb->currDir, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
if (taosMkDir(pSdb->tmpDir) != 0) { if (taosMkDir(pSdb->tmpDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, terrstr()); mError("failed to create dir:%s since %s", pSdb->tmpDir, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
return 0; return 0;

224
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -14,18 +14,19 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "crypt.h"
#include "sdb.h" #include "sdb.h"
#include "sync.h" #include "sync.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "wal.h"
#include "tglobal.h" #include "tglobal.h"
#include "crypt.h" #include "wal.h"
#define SDB_TABLE_SIZE 24 #define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512 #define SDB_RESERVE_SIZE 512
#define SDB_FILE_VER 1 #define SDB_FILE_VER 1
static int32_t sdbDeployData(SSdb *pSdb) { static int32_t sdbDeployData(SSdb *pSdb) {
int32_t code = 0;
mInfo("start to deploy sdb"); mInfo("start to deploy sdb");
for (int32_t i = SDB_MAX - 1; i >= 0; --i) { for (int32_t i = SDB_MAX - 1; i >= 0; --i) {
@ -33,8 +34,9 @@ static int32_t sdbDeployData(SSdb *pSdb) {
if (fp == NULL) continue; if (fp == NULL) continue;
mInfo("start to deploy sdb:%s", sdbTableName(i)); mInfo("start to deploy sdb:%s", sdbTableName(i));
if ((*fp)(pSdb->pMnode) != 0) { code = (*fp)(pSdb->pMnode);
mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr()); if (code != 0) {
mError("failed to deploy sdb:%s since %s", sdbTableName(i), tstrerror(code));
return -1; return -1;
} }
} }
@ -80,61 +82,62 @@ static void sdbResetData(SSdb *pSdb) {
} }
static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = 0; int64_t sver = 0;
int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t)); int32_t ret = taosReadFile(pFile, &sver, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (sver != SDB_FILE_VER) { if (sver != SDB_FILE_VER) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyIndex, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyTerm, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t)); ret = taosReadFile(pFile, &pSdb->applyConfig, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
int64_t maxId = 0; int64_t maxId = 0;
ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); ret = taosReadFile(pFile, &maxId, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (i < SDB_MAX) { if (i < SDB_MAX) {
pSdb->maxId[i] = maxId; pSdb->maxId[i] = maxId;
@ -145,12 +148,12 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
int64_t ver = 0; int64_t ver = 0;
ret = taosReadFile(pFile, &ver, sizeof(int64_t)); ret = taosReadFile(pFile, &ver, sizeof(int64_t));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(int64_t)) { if (ret != sizeof(int64_t)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
if (i < SDB_MAX) { if (i < SDB_MAX) {
pSdb->tableVer[i] = ver; pSdb->tableVer[i] = ver;
@ -160,37 +163,38 @@ static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) {
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
ret = taosReadFile(pFile, reserve, sizeof(reserve)); ret = taosReadFile(pFile, reserve, sizeof(reserve));
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (ret != sizeof(reserve)) { if (ret != sizeof(reserve)) {
terrno = TSDB_CODE_FILE_CORRUPTED; code = TSDB_CODE_FILE_CORRUPTED;
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
} }
static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
int32_t code = 0;
int64_t sver = SDB_FILE_VER; int64_t sver = SDB_FILE_VER;
if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &sver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyIndex, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyTerm, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &pSdb->applyConfig, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) {
@ -199,8 +203,8 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
maxId = pSdb->maxId[i]; maxId = pSdb->maxId[i];
} }
if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
} }
@ -210,15 +214,15 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) {
ver = pSdb->tableVer[i]; ver = pSdb->tableVer[i];
} }
if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
} }
char reserve[SDB_RESERVE_SIZE] = {0}; char reserve[SDB_RESERVE_SIZE] = {0};
if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
return -1; TAOS_RETURN(code);
} }
return 0; return 0;
@ -237,21 +241,22 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100); SSdbRaw *pRaw = taosMemoryMalloc(bufLen + 100);
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since %s", terrstr()); mError("failed read sdb file since %s", tstrerror(code));
return -1; TAOS_RETURN(code);
} }
TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
taosMemoryFree(pRaw); taosMemoryFree(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mInfo("read sdb file:%s finished since %s", file, terrstr()); mInfo("read sdb file:%s finished since %s", file, tstrerror(code));
return 0; return 0;
} }
if (sdbReadFileHead(pSdb, pFile) != 0) { code = sdbReadFileHead(pSdb, pFile);
mError("failed to read sdb file:%s head since %s", file, terrstr()); if (code != 0) {
mError("failed to read sdb file:%s head since %s", file, tstrerror(code));
taosMemoryFree(pRaw); taosMemoryFree(pRaw);
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; return -1;
@ -278,14 +283,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
} }
readLen = pRaw->dataLen + sizeof(int32_t); readLen = pRaw->dataLen + sizeof(int32_t);
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t); readLen = ENCRYPTED_LEN(pRaw->dataLen) + sizeof(int32_t);
} }
if (readLen >= bufLen) { if (readLen >= bufLen) {
bufLen = pRaw->dataLen * 2; bufLen = pRaw->dataLen * 2;
SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100); SSdbRaw *pNewRaw = taosMemoryMalloc(bufLen + 100);
if (pNewRaw == NULL) { if (pNewRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen); mError("failed read sdb file since malloc new sdbRaw size:%d failed", bufLen);
goto _OVER; goto _OVER;
} }
@ -308,10 +313,14 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
goto _OVER; goto _OVER;
} }
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
int32_t count = 0; int32_t count = 0;
char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen)); char *plantContent = taosMemoryMalloc(ENCRYPTED_LEN(pRaw->dataLen));
if (plantContent == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
SCryptOpts opts; SCryptOpts opts;
opts.len = ENCRYPTED_LEN(pRaw->dataLen); opts.len = ENCRYPTED_LEN(pRaw->dataLen);
@ -321,8 +330,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN); strncpy(opts.key, tsEncryptKey, ENCRYPT_KEY_LEN);
count = CBC_Decrypt(&opts); count = CBC_Decrypt(&opts);
//mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__); // mDebug("read sdb, CBC_Decrypt dataLen:%d, descrypted len:%d, %s", pRaw->dataLen, count, __FUNCTION__);
memcpy(pRaw->pData, plantContent, pRaw->dataLen); memcpy(pRaw->pData, plantContent, pRaw->dataLen);
taosMemoryFree(plantContent); taosMemoryFree(plantContent);
@ -355,8 +364,7 @@ _OVER:
taosCloseFile(&pFile); taosCloseFile(&pFile);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
terrno = code; TAOS_RETURN(code);
return code;
} }
int32_t sdbReadFile(SSdb *pSdb) { int32_t sdbReadFile(SSdb *pSdb) {
@ -365,7 +373,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
sdbResetData(pSdb); sdbResetData(pSdb);
int32_t code = sdbReadFileImp(pSdb); int32_t code = sdbReadFileImp(pSdb);
if (code != 0) { if (code != 0) {
mError("failed to read sdb file since %s", terrstr()); mError("failed to read sdb file since %s", tstrerror(code));
sdbResetData(pSdb); sdbResetData(pSdb);
} }
@ -388,13 +396,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s for write since %s", tmpfile, terrstr()); mError("failed to open sdb file:%s for write since %s", tmpfile, tstrerror(code));
return -1; TAOS_RETURN(code);
} }
if (sdbWriteFileHead(pSdb, pFile) != 0) { code = sdbWriteFileHead(pSdb, pFile);
mError("failed to write sdb file:%s head since %s", tmpfile, terrstr()); if (code != 0) {
mError("failed to write sdb file:%s head since %s", tmpfile, tstrerror(code));
taosCloseFile(&pFile); taosCloseFile(&pFile);
return -1; return -1;
} }
@ -436,8 +445,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
} }
int32_t newDataLen = pRaw->dataLen; int32_t newDataLen = pRaw->dataLen;
char* newData = pRaw->pData; char *newData = pRaw->pData;
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
newDataLen = ENCRYPTED_LEN(pRaw->dataLen); newDataLen = ENCRYPTED_LEN(pRaw->dataLen);
newData = taosMemoryMalloc(newDataLen); newData = taosMemoryMalloc(newDataLen);
if (newData == NULL) { if (newData == NULL) {
@ -456,8 +465,8 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
int32_t count = CBC_Encrypt(&opts); int32_t count = CBC_Encrypt(&opts);
//mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s", // mDebug("write sdb, CBC_Encrypt encryptedDataLen:%d, dataLen:%d, %s",
// newDataLen, pRaw->dataLen, __FUNCTION__); // newDataLen, pRaw->dataLen, __FUNCTION__);
} }
if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) { if (taosWriteFile(pFile, newData, newDataLen) != newDataLen) {
@ -467,7 +476,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
break; break;
} }
if(tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB ){ if (tsiEncryptAlgorithm == DND_CA_SM4 && (tsiEncryptScope & DND_CS_SDB) == DND_CS_SDB) {
taosMemoryFree(newData); taosMemoryFree(newData);
} }
@ -549,19 +558,22 @@ int32_t sdbWriteFile(SSdb *pSdb, int32_t delta) {
} }
} }
if (code != 0) { if (code != 0) {
mError("failed to write sdb file since %s", terrstr()); mError("failed to write sdb file since %s", tstrerror(code));
} }
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
return code; return code;
} }
int32_t sdbDeploy(SSdb *pSdb) { int32_t sdbDeploy(SSdb *pSdb) {
if (sdbDeployData(pSdb) != 0) { int32_t code = 0;
return -1; code = sdbDeployData(pSdb);
if (code != 0) {
TAOS_RETURN(code);
} }
if (sdbWriteFile(pSdb, 0) != 0) { code = sdbWriteFile(pSdb, 0);
return -1; if (code != 0) {
TAOS_RETURN(code);
} }
return 0; return 0;
@ -605,6 +617,7 @@ static void sdbCloseIter(SSdbIter *pIter) {
} }
int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) { int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb); SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1; if (pIter == NULL) return -1;
@ -617,19 +630,19 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
int64_t commitConfig = pSdb->commitConfig; int64_t commitConfig = pSdb->commitConfig;
if (taosCopyFile(datafile, pIter->name) < 0) { if (taosCopyFile(datafile, pIter->name) < 0) {
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, terrstr()); mError("failed to copy sdb file %s to %s since %s", datafile, pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
taosThreadMutexUnlock(&pSdb->filelock); taosThreadMutexUnlock(&pSdb->filelock);
pIter->file = taosOpenFile(pIter->name, TD_FILE_READ); pIter->file = taosOpenFile(pIter->name, TD_FILE_READ);
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open sdb file:%s since %s", pIter->name, terrstr()); mError("failed to open sdb file:%s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
*ppIter = pIter; *ppIter = pIter;
@ -645,21 +658,22 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *ter
void sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { sdbCloseIter(pIter); } void sdbStopRead(SSdb *pSdb, SSdbIter *pIter) { sdbCloseIter(pIter); }
int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
int32_t code = 0;
int32_t maxlen = 4096; int32_t maxlen = 4096;
void *pBuf = taosMemoryCalloc(1, maxlen); void *pBuf = taosMemoryCalloc(1, maxlen);
if (pBuf == NULL) { if (pBuf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
return -1; TAOS_RETURN(code);
} }
int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen); int32_t readlen = taosReadFile(pIter->file, pBuf, maxlen);
if (readlen < 0 || readlen > maxlen) { if (readlen < 0 || readlen > maxlen) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, terrstr(), pIter->total); mError("sdbiter:%p, failed to read snapshot since %s, total:%" PRId64, pIter, tstrerror(code), pIter->total);
*ppBuf = NULL; *ppBuf = NULL;
*len = 0; *len = 0;
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
return -1; TAOS_RETURN(code);
} else if (readlen == 0) { } else if (readlen == 0) {
mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total);
*ppBuf = NULL; *ppBuf = NULL;
@ -676,15 +690,19 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) {
} }
int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) {
int32_t code = 0;
SSdbIter *pIter = sdbCreateIter(pSdb); SSdbIter *pIter = sdbCreateIter(pSdb);
if (pIter == NULL) return -1; if (pIter == NULL) {
code = terrno;
TAOS_RETURN(code);
}
pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); pIter->file = taosOpenFile(pIter->name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pIter->file == NULL) { if (pIter->file == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to open %s since %s", pIter->name, terrstr()); mError("failed to open %s since %s", pIter->name, tstrerror(code));
sdbCloseIter(pIter); sdbCloseIter(pIter);
return -1; TAOS_RETURN(code);
} }
*ppIter = pIter; *ppIter = pIter;
@ -702,8 +720,8 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
} }
if (taosFsyncFile(pIter->file) != 0) { if (taosFsyncFile(pIter->file) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, terrstr()); mError("sdbiter:%p, failed to fasync file %s since %s", pIter, pIter->name, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -713,13 +731,14 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply, int64_t index, i
char datafile[PATH_MAX] = {0}; char datafile[PATH_MAX] = {0};
snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); snprintf(datafile, sizeof(datafile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP);
if (taosRenameFile(pIter->name, datafile) != 0) { if (taosRenameFile(pIter->name, datafile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, terrstr()); mError("sdbiter:%p, failed to rename file %s to %s since %s", pIter, pIter->name, datafile, tstrerror(code));
goto _OVER; goto _OVER;
} }
if (sdbReadFile(pSdb) != 0) { code = sdbReadFile(pSdb);
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, terrstr()); if (code != 0) {
mError("sdbiter:%p, failed to read from %s since %s", pIter, datafile, tstrerror(code));
goto _OVER; goto _OVER;
} }
@ -742,11 +761,12 @@ _OVER:
} }
int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) {
int32_t code = 0;
int32_t writelen = taosWriteFile(pIter->file, pBuf, len); int32_t writelen = taosWriteFile(pIter->file, pBuf, len);
if (writelen != len) { if (writelen != len) {
terrno = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
mError("failed to write len:%d since %s, total:%" PRId64, len, terrstr(), pIter->total); mError("failed to write len:%d since %s, total:%" PRId64, len, tstrerror(code), pIter->total);
return -1; TAOS_RETURN(code);
} }
pIter->total += writelen; pIter->total += writelen;

131
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -53,14 +53,15 @@ void sdbFreeRaw(SSdbRaw *pRaw) {
} }
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int8_t) > pRaw->dataLen) { if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int8_t *)(pRaw->pData + dataPos) = val; *(int8_t *)(pRaw->pData + dataPos) = val;
@ -68,14 +69,15 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
} }
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) { int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(uint8_t *)(pRaw->pData + dataPos) = val; *(uint8_t *)(pRaw->pData + dataPos) = val;
@ -83,14 +85,15 @@ int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
} }
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int32_t) > pRaw->dataLen) { if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int32_t *)(pRaw->pData + dataPos) = val; *(int32_t *)(pRaw->pData + dataPos) = val;
@ -98,14 +101,15 @@ int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
} }
int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) { int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int16_t) > pRaw->dataLen) { if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int16_t *)(pRaw->pData + dataPos) = val; *(int16_t *)(pRaw->pData + dataPos) = val;
@ -113,14 +117,15 @@ int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val) {
} }
int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) { int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int64_t) > pRaw->dataLen) { if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*(int64_t *)(pRaw->pData + dataPos) = val; *(int64_t *)(pRaw->pData + dataPos) = val;
@ -128,14 +133,15 @@ int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val) {
} }
int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) { int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + valLen > pRaw->dataLen) { if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
if (pVal != NULL) { if (pVal != NULL) {
@ -145,14 +151,15 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
} }
int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) { int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataLen > pRaw->dataLen) { if (dataLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
pRaw->dataLen = dataLen; pRaw->dataLen = dataLen;
@ -160,14 +167,15 @@ int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen) {
} }
int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) { int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (status == SDB_STATUS_INIT) { if (status == SDB_STATUS_INIT) {
terrno = TSDB_CODE_INVALID_PARA; code = TSDB_CODE_INVALID_PARA;
return -1; TAOS_RETURN(code);
} }
pRaw->status = status; pRaw->status = status;
@ -175,14 +183,15 @@ int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status) {
} }
int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) { int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int8_t) > pRaw->dataLen) { if (dataPos + sizeof(int8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int8_t *)(pRaw->pData + dataPos); *val = *(int8_t *)(pRaw->pData + dataPos);
@ -190,14 +199,15 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
} }
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) { int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(uint8_t *)(pRaw->pData + dataPos); *val = *(uint8_t *)(pRaw->pData + dataPos);
@ -205,14 +215,15 @@ int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
} }
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int32_t) > pRaw->dataLen) { if (dataPos + sizeof(int32_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int32_t *)(pRaw->pData + dataPos); *val = *(int32_t *)(pRaw->pData + dataPos);
@ -220,14 +231,15 @@ int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
} }
int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) { int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int16_t) > pRaw->dataLen) { if (dataPos + sizeof(int16_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int16_t *)(pRaw->pData + dataPos); *val = *(int16_t *)(pRaw->pData + dataPos);
@ -235,14 +247,15 @@ int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val) {
} }
int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) { int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + sizeof(int64_t) > pRaw->dataLen) { if (dataPos + sizeof(int64_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
*val = *(int64_t *)(pRaw->pData + dataPos); *val = *(int64_t *)(pRaw->pData + dataPos);
@ -250,14 +263,15 @@ int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val) {
} }
int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) { int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valLen) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
if (dataPos + valLen > pRaw->dataLen) { if (dataPos + valLen > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; code = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1; TAOS_RETURN(code);
} }
if (pVal != NULL) { if (pVal != NULL) {
memcpy(pVal, pRaw->pData + dataPos, valLen); memcpy(pVal, pRaw->pData + dataPos, valLen);
@ -266,9 +280,10 @@ int32_t sdbGetRawBinary(SSdbRaw *pRaw, int32_t dataPos, char *pVal, int32_t valL
} }
int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver) { int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver) {
int32_t code = 0;
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; code = TSDB_CODE_INVALID_PTR;
return -1; TAOS_RETURN(code);
} }
*sver = pRaw->sver; *sver = pRaw->sver;

2
source/dnode/vnode/inc/vnode.h
View File

@ -167,7 +167,7 @@ void tsdbReaderClose2(STsdbReader *pReader);
int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext); int32_t tsdbNextDataBlock2(STsdbReader *pReader, bool *hasNext);
int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); int32_t tsdbRetrieveDatablockSMA2(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
void tsdbReleaseDataBlock2(STsdbReader *pReader); void tsdbReleaseDataBlock2(STsdbReader *pReader);
SSDataBlock *tsdbRetrieveDataBlock2(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbRetrieveDataBlock2(STsdbReader *pReader, SSDataBlock **pBlock, SArray *pIdList);
int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbReaderReset2(STsdbReader *pReader, SQueryTableDataCond *pCond);
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo2(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo);
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle); int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader *pHandle);

155
source/dnode/vnode/src/inc/tsdb.h
View File

@ -147,7 +147,6 @@ int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key); void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key); void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
// STSDBRowIter // STSDBRowIter
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
void tsdbRowClose(STSDBRowIter *pIter); void tsdbRowClose(STSDBRowIter *pIter);
@ -245,10 +244,10 @@ void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive);
STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode);
int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive);
SArray * tsdbMemTableGetTbDataArray(SMemTable *pMemTable); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
// STbDataIter // STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter); int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter);
void * tsdbTbDataIterDestroy(STbDataIter *pIter); void *tsdbTbDataIterDestroy(STbDataIter *pIter);
void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter *pIter);
bool tsdbTbDataIterNext(STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter);
void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum); void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum);
@ -301,7 +300,7 @@ int32_t tsdbGetTableSchema(SMeta *pMeta, int64_t uid, STSchema **pSchema, int64_
// tsdbMerge.c ============================================================================================== // tsdbMerge.c ==============================================================================================
typedef struct { typedef struct {
STsdb * tsdb; STsdb *tsdb;
int32_t fid; int32_t fid;
} SMergeArg; } SMergeArg;
@ -332,22 +331,22 @@ int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo);
// structs ======================= // structs =======================
struct STsdbFS { struct STsdbFS {
SDelFile *pDelFile; SDelFile *pDelFile;
SArray * aDFileSet; // SArray<SDFileSet> SArray *aDFileSet; // SArray<SDFileSet>
}; };
typedef struct { typedef struct {
rocksdb_t * db; rocksdb_t *db;
rocksdb_comparator_t * my_comparator; rocksdb_comparator_t *my_comparator;
rocksdb_cache_t * blockcache; rocksdb_cache_t *blockcache;
rocksdb_block_based_table_options_t *tableoptions; rocksdb_block_based_table_options_t *tableoptions;
rocksdb_options_t * options; rocksdb_options_t *options;
rocksdb_flushoptions_t * flushoptions; rocksdb_flushoptions_t *flushoptions;
rocksdb_writeoptions_t * writeoptions; rocksdb_writeoptions_t *writeoptions;
rocksdb_readoptions_t * readoptions; rocksdb_readoptions_t *readoptions;
rocksdb_writebatch_t * writebatch; rocksdb_writebatch_t *writebatch;
rocksdb_writebatch_t * rwritebatch; rocksdb_writebatch_t *rwritebatch;
TdThreadMutex rMutex; TdThreadMutex rMutex;
STSchema * pTSchema; STSchema *pTSchema;
} SRocksCache; } SRocksCache;
typedef struct { typedef struct {
@ -358,26 +357,26 @@ typedef struct {
typedef struct SCompMonitor SCompMonitor; typedef struct SCompMonitor SCompMonitor;
struct STsdb { struct STsdb {
char * path; char *path;
SVnode * pVnode; SVnode *pVnode;
STsdbKeepCfg keepCfg; STsdbKeepCfg keepCfg;
TdThreadMutex mutex; TdThreadMutex mutex;
bool bgTaskDisabled; bool bgTaskDisabled;
SMemTable * mem; SMemTable *mem;
SMemTable * imem; SMemTable *imem;
STsdbFS fs; // old STsdbFS fs; // old
SLRUCache * lruCache; SLRUCache *lruCache;
SCacheFlushState flushState; SCacheFlushState flushState;
TdThreadMutex lruMutex; TdThreadMutex lruMutex;
SLRUCache * biCache; SLRUCache *biCache;
TdThreadMutex biMutex; TdThreadMutex biMutex;
SLRUCache * bCache; SLRUCache *bCache;
TdThreadMutex bMutex; TdThreadMutex bMutex;
SLRUCache * pgCache; SLRUCache *pgCache;
TdThreadMutex pgMutex; TdThreadMutex pgMutex;
struct STFileSystem *pFS; // new struct STFileSystem *pFS; // new
SRocksCache rCache; SRocksCache rCache;
SCompMonitor *pCompMonitor; SCompMonitor *pCompMonitor;
struct { struct {
SVHashTable *ht; SVHashTable *ht;
SArray *arr; SArray *arr;
@ -405,17 +404,17 @@ struct STbData {
TSKEY minKey; TSKEY minKey;
TSKEY maxKey; TSKEY maxKey;
SRWLatch lock; SRWLatch lock;
SDelData * pHead; SDelData *pHead;
SDelData * pTail; SDelData *pTail;
SMemSkipList sl; SMemSkipList sl;
STbData * next; STbData *next;
SRBTreeNode rbtn[1]; SRBTreeNode rbtn[1];
}; };
struct SMemTable { struct SMemTable {
SRWLatch latch; SRWLatch latch;
STsdb * pTsdb; STsdb *pTsdb;
SVBufPool * pPool; SVBufPool *pPool;
volatile int32_t nRef; volatile int32_t nRef;
int64_t minVer; int64_t minVer;
int64_t maxVer; int64_t maxVer;
@ -425,7 +424,7 @@ struct SMemTable {
int64_t nDel; int64_t nDel;
int32_t nTbData; int32_t nTbData;
int32_t nBucket; int32_t nBucket;
STbData ** aBucket; STbData **aBucket;
SRBTree tbDataTree[1]; SRBTree tbDataTree[1];
}; };
@ -434,7 +433,7 @@ struct TSDBROW {
union { union {
struct { struct {
int64_t version; int64_t version;
SRow * pTSRow; SRow *pTSRow;
}; };
struct { struct {
SBlockData *pBlockData; SBlockData *pBlockData;
@ -535,9 +534,9 @@ struct SBlockData {
int64_t suid; // 0 means normal table block data, otherwise child table block data int64_t suid; // 0 means normal table block data, otherwise child table block data
int64_t uid; // 0 means block data in .last file, otherwise in .data file int64_t uid; // 0 means block data in .last file, otherwise in .data file
int32_t nRow; // number of rows int32_t nRow; // number of rows
int64_t * aUid; // uids of each row, only exist in block data in .last file (uid == 0) int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0)
int64_t * aVersion; // versions of each row int64_t *aVersion; // versions of each row
TSKEY * aTSKEY; // timestamp of each row TSKEY *aTSKEY; // timestamp of each row
int32_t nColData; int32_t nColData;
SColData *aColData; SColData *aColData;
}; };
@ -548,10 +547,10 @@ struct TABLEID {
}; };
struct STbDataIter { struct STbDataIter {
STbData * pTbData; STbData *pTbData;
int8_t backward; int8_t backward;
SMemSkipListNode *pNode; SMemSkipListNode *pNode;
TSDBROW * pRow; TSDBROW *pRow;
TSDBROW row; TSDBROW row;
}; };
@ -629,9 +628,9 @@ struct SDFileSet {
int32_t fid; int32_t fid;
SHeadFile *pHeadF; SHeadFile *pHeadF;
SDataFile *pDataF; SDataFile *pDataF;
SSmaFile * pSmaF; SSmaFile *pSmaF;
uint8_t nSttF; uint8_t nSttF;
SSttFile * aSttF[TSDB_STT_TRIGGER_ARRAY_SIZE]; SSttFile *aSttF[TSDB_STT_TRIGGER_ARRAY_SIZE];
}; };
struct STSDBRowIter { struct STSDBRowIter {
@ -647,18 +646,18 @@ struct STSDBRowIter {
struct SRowMerger { struct SRowMerger {
STSchema *pTSchema; STSchema *pTSchema;
int64_t version; int64_t version;
SArray * pArray; // SArray<SColVal> SArray *pArray; // SArray<SColVal>
}; };
typedef struct { typedef struct {
char * path; char *path;
int32_t szPage; int32_t szPage;
int32_t flag; int32_t flag;
TdFilePtr pFD; TdFilePtr pFD;
int64_t pgno; int64_t pgno;
uint8_t * pBuf; uint8_t *pBuf;
int64_t szFile; int64_t szFile;
STsdb * pTsdb; STsdb *pTsdb;
const char *objName; const char *objName;
uint8_t s3File; uint8_t s3File;
int32_t lcn; int32_t lcn;
@ -668,7 +667,7 @@ typedef struct {
} STsdbFD; } STsdbFD;
struct SDelFWriter { struct SDelFWriter {
STsdb * pTsdb; STsdb *pTsdb;
SDelFile fDel; SDelFile fDel;
STsdbFD *pWriteH; STsdbFD *pWriteH;
uint8_t *aBuf[1]; uint8_t *aBuf[1];
@ -728,15 +727,15 @@ int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
// snap read // snap read
struct STsdbReadSnap { struct STsdbReadSnap {
SMemTable * pMem; SMemTable *pMem;
SQueryNode * pNode; SQueryNode *pNode;
SMemTable * pIMem; SMemTable *pIMem;
SQueryNode * pINode; SQueryNode *pINode;
TFileSetArray *pfSetArray; TFileSetArray *pfSetArray;
}; };
struct SDataFWriter { struct SDataFWriter {
STsdb * pTsdb; STsdb *pTsdb;
SDFileSet wSet; SDFileSet wSet;
STsdbFD *pHeadFD; STsdbFD *pHeadFD;
@ -753,13 +752,13 @@ struct SDataFWriter {
}; };
struct SDataFReader { struct SDataFReader {
STsdb * pTsdb; STsdb *pTsdb;
SDFileSet *pSet; SDFileSet *pSet;
STsdbFD * pHeadFD; STsdbFD *pHeadFD;
STsdbFD * pDataFD; STsdbFD *pDataFD;
STsdbFD * pSmaFD; STsdbFD *pSmaFD;
STsdbFD * aSttFD[TSDB_STT_TRIGGER_ARRAY_SIZE]; STsdbFD *aSttFD[TSDB_STT_TRIGGER_ARRAY_SIZE];
uint8_t * aBuf[3]; uint8_t *aBuf[3];
}; };
// NOTE: do NOT change the order of the fields // NOTE: do NOT change the order of the fields
@ -794,10 +793,10 @@ typedef struct {
typedef struct SSttBlockLoadInfo { typedef struct SSttBlockLoadInfo {
SBlockDataInfo blockData[2]; // buffered block data SBlockDataInfo blockData[2]; // buffered block data
SArray * aSttBlk; SArray *aSttBlk;
int32_t currentLoadBlockIndex; int32_t currentLoadBlockIndex;
STSchema * pSchema; STSchema *pSchema;
int16_t * colIds; int16_t *colIds;
int32_t numOfCols; int32_t numOfCols;
bool checkRemainingRow; // todo: no assign value? bool checkRemainingRow; // todo: no assign value?
bool isLast; bool isLast;
@ -834,7 +833,7 @@ struct SDiskData {
const uint8_t *pUid; const uint8_t *pUid;
const uint8_t *pVer; const uint8_t *pVer;
const uint8_t *pKey; const uint8_t *pKey;
SArray * aDiskCol; // SArray<SDiskCol> SArray *aDiskCol; // SArray<SDiskCol>
}; };
struct SDiskDataBuilder { struct SDiskDataBuilder {
@ -847,15 +846,15 @@ struct SDiskDataBuilder {
SCompressor *pVerC; SCompressor *pVerC;
SCompressor *pKeyC; SCompressor *pKeyC;
int32_t nBuilder; int32_t nBuilder;
SArray * aBuilder; // SArray<SDiskColBuilder> SArray *aBuilder; // SArray<SDiskColBuilder>
uint8_t * aBuf[2]; uint8_t *aBuf[2];
SDiskData dd; SDiskData dd;
SBlkInfo bi; SBlkInfo bi;
}; };
struct SLDataIter { struct SLDataIter {
SRBTreeNode node; SRBTreeNode node;
SSttBlk * pSttBlk; SSttBlk *pSttBlk;
int64_t cid; // for debug purpose int64_t cid; // for debug purpose
int8_t backward; int8_t backward;
int32_t iSttBlk; int32_t iSttBlk;
@ -864,8 +863,8 @@ struct SLDataIter {
uint64_t uid; uint64_t uid;
STimeWindow timeWindow; STimeWindow timeWindow;
SVersionRange verRange; SVersionRange verRange;
SSttBlockLoadInfo * pBlockLoadInfo; SSttBlockLoadInfo *pBlockLoadInfo;
SRowKey * pStartRowKey; // current row key SRowKey *pStartRowKey; // current row key
bool ignoreEarlierTs; bool ignoreEarlierTs;
struct SSttFileReader *pReader; struct SSttFileReader *pReader;
}; };
@ -878,21 +877,21 @@ typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pS
typedef struct SMergeTreeConf { typedef struct SMergeTreeConf {
int8_t backward; int8_t backward;
STsdb * pTsdb; STsdb *pTsdb;
uint64_t suid; uint64_t suid;
uint64_t uid; uint64_t uid;
STimeWindow timewindow; STimeWindow timewindow;
SVersionRange verRange; SVersionRange verRange;
bool strictTimeRange; bool strictTimeRange;
SArray * pSttFileBlockIterArray; SArray *pSttFileBlockIterArray;
void * pCurrentFileset; void *pCurrentFileset;
STSchema * pSchema; STSchema *pSchema;
int16_t * pCols; int16_t *pCols;
int32_t numOfCols; int32_t numOfCols;
SRowKey * pCurRowKey; SRowKey *pCurRowKey;
_load_tomb_fn loadTombFn; _load_tomb_fn loadTombFn;
void * pReader; void *pReader;
void * idstr; void *idstr;
bool rspRows; // response the rows in stt-file, if possible bool rspRows; // response the rows in stt-file, if possible
} SMergeTreeConf; } SMergeTreeConf;
@ -909,9 +908,9 @@ void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo);
void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ============================================================================================== // tsdbCache ==============================================================================================
typedef enum { typedef enum {
@ -1023,7 +1022,7 @@ struct STsdbDataIter2 {
// TSDB_DATA_FILE_DATA_ITER // TSDB_DATA_FILE_DATA_ITER
struct { struct {
SDataFReader *pReader; SDataFReader *pReader;
SArray * aBlockIdx; // SArray<SBlockIdx> SArray *aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk; SMapData mDataBlk;
SBlockData bData; SBlockData bData;
int32_t iBlockIdx; int32_t iBlockIdx;
@ -1035,7 +1034,7 @@ struct STsdbDataIter2 {
struct { struct {
SDataFReader *pReader; SDataFReader *pReader;
int32_t iStt; int32_t iStt;
SArray * aSttBlk; SArray *aSttBlk;
SBlockData bData; SBlockData bData;
int32_t iSttBlk; int32_t iSttBlk;
int32_t iRow; int32_t iRow;
@ -1043,8 +1042,8 @@ struct STsdbDataIter2 {
// TSDB_TOMB_FILE_DATA_ITER // TSDB_TOMB_FILE_DATA_ITER
struct { struct {
SDelFReader *pReader; SDelFReader *pReader;
SArray * aDelIdx; SArray *aDelIdx;
SArray * aDelData; SArray *aDelData;
int32_t iDelIdx; int32_t iDelIdx;
int32_t iDelData; int32_t iDelData;
} tIter; } tIter;

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -54,7 +54,6 @@ extern "C" {
#endif #endif
typedef struct SVnodeInfo SVnodeInfo; typedef struct SVnodeInfo SVnodeInfo;
typedef struct SMeta SMeta;
typedef struct SSma SSma; typedef struct SSma SSma;
typedef struct STsdb STsdb; typedef struct STsdb STsdb;
typedef struct STQ STQ; typedef struct STQ STQ;
@ -153,7 +152,6 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode);
void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive);
// meta // meta
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMStbCursor SMStbCursor; typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore; typedef struct STbUidStore STbUidStore;

81
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -61,7 +61,7 @@ struct SRSmaQTaskInfoItem {
int32_t len; int32_t len;
int8_t type; int8_t type;
int64_t suid; int64_t suid;
void * qTaskInfo; void *qTaskInfo;
}; };
static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) {
@ -185,7 +185,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) {
void *pIter = NULL; void *pIter = NULL;
while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { while ((pIter = taosHashIterate(pStore->uidHash, pIter))) {
tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL);
SArray * pTbUids = *(SArray **)pIter; SArray *pTbUids = *(SArray **)pIter;
if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) { if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) {
taosHashCancelIterate(pStore->uidHash, pIter); taosHashCancelIterate(pStore->uidHash, pIter);
@ -213,7 +213,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui
} }
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SHashObj * infoHash = NULL; SHashObj *infoHash = NULL;
if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) {
terrno = TSDB_CODE_RSMA_INVALID_STAT; terrno = TSDB_CODE_RSMA_INVALID_STAT;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
@ -264,11 +264,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
int8_t idx) { int8_t idx) {
if ((param->qmsgLen > 0) && param->qmsg[idx]) { if ((param->qmsgLen > 0) && param->qmsg[idx]) {
SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]);
SRetention * pRetention = SMA_RETENTION(pSma); SRetention *pRetention = SMA_RETENTION(pSma);
STsdbCfg * pTsdbCfg = SMA_TSDB_CFG(pSma); STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma);
SVnode * pVnode = pSma->pVnode; SVnode *pVnode = pSma->pVnode;
char taskInfDir[TSDB_FILENAME_LEN] = {0}; char taskInfDir[TSDB_FILENAME_LEN] = {0};
void * pStreamState = NULL; void *pStreamState = NULL;
// set the backend of stream state // set the backend of stream state
tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir);
@ -362,6 +362,8 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
* @return int32_t * @return int32_t
*/ */
int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) {
int32_t code;
if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) {
smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -374,7 +376,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
} }
#endif #endif
SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SRSmaInfo *pRSmaInfo = NULL; SRSmaInfo *pRSmaInfo = NULL;
@ -401,13 +403,18 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
pRSmaInfo->suid = suid; pRSmaInfo->suid = suid;
T_REF_INIT_VAL(pRSmaInfo, 1); T_REF_INIT_VAL(pRSmaInfo, 1);
if (!(pRSmaInfo->queue = taosOpenQueue()) || !(pRSmaInfo->qall = taosAllocateQall()) || code = taosOpenQueue(&pRSmaInfo->queue);
tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 || if (code) goto _err;
code = taosAllocateQall(&pRSmaInfo->qall);
if (code) goto _err;
if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0 ||
tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) { tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 1) < 0) {
goto _err; goto _err;
} }
if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != 0) {
goto _err; goto _err;
} }
@ -533,12 +540,12 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid)
taosArrayDestroy(pUidArray); taosArrayDestroy(pUidArray);
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
} else { } else {
if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) < 0) { if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
} }
@ -673,8 +680,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SSDataBlock *output = NULL; SSDataBlock *output = NULL;
SArray * pResList = pItem->pResList; SArray *pResList = pItem->pResList;
STSchema * pTSchema = pInfo->pTSchema; STSchema *pTSchema = pInfo->pTSchema;
int64_t suid = pInfo->suid; int64_t suid = pInfo->suid;
while (1) { while (1) {
@ -733,7 +740,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
} }
} }
STsdb * sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
SSubmitReq2 *pReq = NULL; SSubmitReq2 *pReq = NULL;
if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) {
@ -795,11 +802,13 @@ _exit:
*/ */
static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType,
SRSmaInfo *pInfo, tb_uid_t suid) { SRSmaInfo *pInfo, tb_uid_t suid) {
int32_t code;
int32_t size = RSMA_EXEC_MSG_HLEN + len; // header + payload int32_t size = RSMA_EXEC_MSG_HLEN + len; // header + payload
void * qItem = taosAllocateQitem(size, DEF_QITEM, 0); void *qItem;
if (!qItem) { code = taosAllocateQitem(size, DEF_QITEM, 0, (void **)&qItem);
return TSDB_CODE_FAILED; if (code) {
return code;
} }
void *pItem = qItem; void *pItem = qItem;
@ -874,7 +883,7 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) {
static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType, static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int64_t version, int32_t inputType,
SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) {
int32_t idx = level - 1; int32_t idx = level - 1;
void * qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx);
SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx);
if (!qTaskInfo) { if (!qTaskInfo) {
@ -914,7 +923,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize,
static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pStat = NULL; SRSmaStat *pStat = NULL;
SRSmaInfo *pRSmaInfo = NULL; SRSmaInfo *pRSmaInfo = NULL;
@ -1069,8 +1078,8 @@ _err:
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SVnode * pVnode = pSma->pVnode; SVnode *pVnode = pSma->pVnode;
SArray * suidList = NULL; SArray *suidList = NULL;
STbUidStore uidStore = {0}; STbUidStore uidStore = {0};
SMetaReader mr = {0}; SMetaReader mr = {0};
tb_uid_t suid = 0; tb_uid_t suid = 0;
@ -1198,7 +1207,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t nTaskInfo = 0; int32_t nTaskInfo = 0;
SSma * pSma = pRSmaStat->pSma; SSma *pSma = pRSmaStat->pSma;
SVnode *pVnode = pSma->pVnode; SVnode *pVnode = pSma->pVnode;
if (taosHashGetSize(pInfoHash) <= 0) { if (taosHashGetSize(pInfoHash) <= 0) {
@ -1231,7 +1240,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) {
do { do {
int32_t nStreamFlushed = 0; int32_t nStreamFlushed = 0;
int32_t nSleep = 0; int32_t nSleep = 0;
void * infoHash = NULL; void *infoHash = NULL;
while (true) { while (true) {
while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
@ -1273,7 +1282,7 @@ _checkpoint:
SStreamMeta *pMeta = NULL; SStreamMeta *pMeta = NULL;
int64_t checkpointId = taosGetTimestampNs(); int64_t checkpointId = taosGetTimestampNs();
bool checkpointBuilt = false; bool checkpointBuilt = false;
void * infoHash = NULL; void *infoHash = NULL;
while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { while ((infoHash = taosHashIterate(pInfoHash, infoHash))) {
SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash;
if (RSMA_INFO_IS_DEL(pRSmaInfo)) { if (RSMA_INFO_IS_DEL(pRSmaInfo)) {
@ -1346,10 +1355,10 @@ _exit:
* @param tmrId * @param tmrId
*/ */
static void tdRSmaFetchTrigger(void *param, void *tmrId) { static void tdRSmaFetchTrigger(void *param, void *tmrId) {
SRSmaRef * pRSmaRef = NULL; SRSmaRef *pRSmaRef = NULL;
SSma * pSma = NULL; SSma *pSma = NULL;
SRSmaStat * pStat = NULL; SRSmaStat *pStat = NULL;
SRSmaInfo * pRSmaInfo = NULL; SRSmaInfo *pRSmaInfo = NULL;
SRSmaInfoItem *pItem = NULL; SRSmaInfoItem *pItem = NULL;
if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, &param, POINTER_BYTES))) { if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, &param, POINTER_BYTES))) {
@ -1517,7 +1526,7 @@ _err:
} }
static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) { static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SArray *pSubmitArr, ERsmaExecType type) {
void * msg = NULL; void *msg = NULL;
int8_t resume = 0; int8_t resume = 0;
int32_t nSubmit = 0; int32_t nSubmit = 0;
int32_t nDelete = 0; int32_t nDelete = 0;
@ -1548,7 +1557,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
_resume_delete: _resume_delete:
version = RSMA_EXEC_MSG_VER(msg); version = RSMA_EXEC_MSG_VER(msg);
if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version, if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version,
&packData.pDataBlock, 1))) { &packData.pDataBlock, 1))) {
taosFreeQitem(msg); taosFreeQitem(msg);
goto _err; goto _err;
} }
@ -1621,11 +1630,11 @@ _err:
int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SVnode * pVnode = pSma->pVnode; SVnode *pVnode = pSma->pVnode;
SSmaEnv * pEnv = SMA_RSMA_ENV(pSma); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma);
SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
SHashObj * infoHash = NULL; SHashObj *infoHash = NULL;
SArray * pSubmitArr = NULL; SArray *pSubmitArr = NULL;
bool isFetchAll = false; bool isFetchAll = false;
if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) {

2
source/dnode/vnode/src/tq/tq.c
View File

@ -612,7 +612,7 @@ int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t
return -1; return -1;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) < 0) { if (taosHashPut(pTq->pCheckInfo, info.topic, strlen(info.topic), &info, sizeof(STqCheckInfo)) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }

2
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -74,7 +74,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) != 0) {
return -1; return -1;
} }

118
source/dnode/vnode/src/tq/tqSink.c
View File

@ -46,6 +46,7 @@ static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid,
static SArray* createDefaultTagColName(); static SArray* createDefaultTagColName();
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName, static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule); int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr, bool newSubTableRule) { const char* pIdStr, bool newSubTableRule) {
@ -270,6 +271,14 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayPush(reqs.pArray, pCreateTbReq); taosArrayPush(reqs.pArray, pCreateTbReq);
STableSinkInfo* pInfo = NULL;
bool alreadyCached = tqGetTableInfo(pTask->outputInfo.tbSink.pTblInfo, gid, &pInfo);
if (!alreadyCached) {
code = doCreateSinkInfo(pCreateTbReq->name, &pInfo);
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pInfo, gid, pTask->id.idStr);
}
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
} }
@ -634,6 +643,18 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo) {
int32_t nameLen = strlen(pDstTableName);
(*pInfo) = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1);
if (*pInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pInfo)->name.len = nameLen;
memcpy((*pInfo)->name.data, pDstTableName, nameLen);
return TSDB_CODE_SUCCESS;
}
int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
SSubmitTbData* pTableData) { SSubmitTbData* pTableData) {
uint64_t groupId = pDataBlock->info.id.groupId; uint64_t groupId = pDataBlock->info.id.groupId;
@ -670,22 +691,15 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) &&
!alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) {
tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName);
if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) {
buildCtbNameAddGroupId(NULL, dstTableName, groupId); buildCtbNameAddGroupId(NULL, dstTableName, groupId);
}else if(pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) { } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER && stbFullName) {
buildCtbNameAddGroupId(stbFullName, dstTableName, groupId); buildCtbNameAddGroupId(stbFullName, dstTableName, groupId);
} }
} }
} }
int32_t nameLen = strlen(dstTableName); int32_t code = doCreateSinkInfo(dstTableName, &pTableSinkInfo);
pTableSinkInfo = taosMemoryCalloc(1, sizeof(STableSinkInfo) + nameLen + 1);
if (pTableSinkInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pTableSinkInfo->name.len = nameLen;
memcpy(pTableSinkInfo->name.data, dstTableName, nameLen);
tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName); tqDebug("s-task:%s build new sinkTableInfo to add cache, dstTable:%s", id, dstTableName);
} }
@ -693,7 +707,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
pTableData->uid = pTableSinkInfo->uid; pTableData->uid = pTableSinkInfo->uid;
if (pTableData->uid == 0) { if (pTableData->uid == 0) {
tqTrace("s-task:%s cached tableInfo uid is invalid, acquire it from meta", id); tqTrace("s-task:%s cached tableInfo:%s uid is invalid, acquire it from meta", id, pTableSinkInfo->name.data);
return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid); return doWaitForDstTableCreated(pVnode, pTask, pTableSinkInfo, dstTableName, &pTableData->uid);
} else { } else {
tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid); tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid);
@ -711,24 +725,33 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
if (metaGetTableEntryByName(&mr, dstTableName) < 0) { if (metaGetTableEntryByName(&mr, dstTableName) < 0) {
metaReaderClear(&mr); metaReaderClear(&mr);
tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); if (pTask->outputInfo.tbSink.autoCreateCtb) {
tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = pTableData->pCreateTbReq =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray, buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
taosArrayDestroy(pTagArray); taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (pTableData->pCreateTbReq == NULL) {
tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); tqError("s-task:%s failed to build auto create dst-table req:%s, code:%s", id, dstTableName,
taosMemoryFree(pTableSinkInfo); tstrerror(terrno));
return terrno; taosMemoryFree(pTableSinkInfo);
return terrno;
}
pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else {
metaReaderClear(&mr);
tqError("s-task:%s vgId:%d dst-table:%s not auto-created, and not create in tsdb, discard data", id,
vgId, dstTableName);
return TSDB_CODE_TDB_TABLE_NOT_EXIST;
} }
pTableSinkInfo->uid = 0;
doPutIntoCache(pTask->outputInfo.tbSink.pTblInfo, pTableSinkInfo, groupId, id);
} else { } else {
bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid); bool isValid = isValidDstChildTable(&mr, vgId, dstTableName, suid);
if (!isValid) { if (!isValid) {
@ -774,16 +797,34 @@ int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_
} }
void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
const SArray* pBlocks = (const SArray*)data; const SArray* pBlocks = (const SArray*)data;
SVnode* pVnode = (SVnode*)vnode; SVnode* pVnode = (SVnode*)vnode;
int64_t suid = pTask->outputInfo.tbSink.stbUid; int64_t suid = pTask->outputInfo.tbSink.stbUid;
char* stbFullName = pTask->outputInfo.tbSink.stbFullName; char* stbFullName = pTask->outputInfo.tbSink.stbFullName;
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t vgId = TD_VID(pVnode); int32_t vgId = TD_VID(pVnode);
int32_t numOfBlocks = taosArrayGetSize(pBlocks); int32_t numOfBlocks = taosArrayGetSize(pBlocks);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb);
STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
if (pTask->outputInfo.tbSink.pTagSchema == NULL) {
SMetaReader mer1 = {0};
metaReaderDoInit(&mer1, pVnode->pMeta, META_READER_LOCK);
code = metaReaderGetTableEntryByUid(&mer1, pOutputInfo->tbSink.stbUid);
pOutputInfo->tbSink.pTagSchema = tCloneSSchemaWrapper(&mer1.me.stbEntry.schemaTag);
metaReaderClear(&mer1);
SSchemaWrapper* pTagSchema = pOutputInfo->tbSink.pTagSchema;
SSchema* pCol1 = &pTagSchema->pSchema[0];
if (pTagSchema->nCols == 1 && pCol1->type == TSDB_DATA_TYPE_UBIGINT && strcmp(pCol1->name, "group_id") == 0) {
pOutputInfo->tbSink.autoCreateCtb = true;
} else {
pOutputInfo->tbSink.autoCreateCtb = false;
}
}
bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks);
if (!onlySubmitData) { if (!onlySubmitData) {
@ -815,6 +856,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP}; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = TD_REQ_FROM_APP};
code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("vgId:%d s-task:%s dst-table not exist, stb:%s discard stream results", vgId, id, stbFullName);
continue; continue;
} }
@ -868,6 +910,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
if (index == NULL) { // no data yet, append it if (index == NULL) { // no data yet, append it
code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData); code = setDstTableDataUid(pVnode, pTask, pDataBlock, stbFullName, &tbData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("vgId:%d dst-table gid:%" PRId64 " not exist, discard stream results", vgId, groupId);
continue; continue;
} }
@ -929,11 +972,6 @@ bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) {
} }
int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) {
if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) {
taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it
return TSDB_CODE_FAILED;
}
int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pTableSinkInfo); taosMemoryFreeClear(pTableSinkInfo);

66
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -164,7 +164,8 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
taosWUnLockLatch(&pTq->lock); taosWUnLockLatch(&pTq->lock);
} }
tOffsetCopy(&dataRsp.common.reqOffset, pOffset); // reqOffset represents the current date offset, may be changed if wal not exists tOffsetCopy(&dataRsp.common.reqOffset,
pOffset); // reqOffset represents the current date offset, may be changed if wal not exists
code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
end : { end : {
@ -178,25 +179,25 @@ end : {
} }
} }
#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \ #define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \
SDecoder decoder = {0};\ SDecoder decoder = {0}; \
TYPE req = {0}; \ TYPE req = {0}; \
void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \
int32_t len = pHead->bodyLen - sizeof(SMsgHead); \ int32_t len = pHead->bodyLen - sizeof(SMsgHead); \
tDecoderInit(&decoder, data, len); \ tDecoderInit(&decoder, data, len); \
if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \ if (DECODE_FUNC(&decoder, &req) == 0 && (req.source & TD_REQ_FROM_TAOX) != 0) { \
tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \ tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 \
pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ " msgType %d", \
fetchVer++; \ pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \
DELETE_FUNC(&req); \ fetchVer++; \
tDecoderClear(&decoder); \ DELETE_FUNC(&req); \
continue; \ tDecoderClear(&decoder); \
} \ continue; \
DELETE_FUNC(&req); \ } \
DELETE_FUNC(&req); \
tDecoderClear(&decoder); tDecoderClear(&decoder);
static void tDeleteCommon(void* parm) { static void tDeleteCommon(void* parm) {}
}
static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest,
SRpcMsg* pMsg, STqOffsetVal* offset) { SRpcMsg* pMsg, STqOffsetVal* offset) {
@ -313,9 +314,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle,
tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); tqError("tmq extract meta from log, tEncodeMqMetaRsp error");
continue; continue;
} }
int32_t tLen = sizeof(SMqRspHead) + len; int32_t tLen = sizeof(SMqRspHead) + len;
void* tBuf = taosMemoryCalloc(1, tLen); void* tBuf = taosMemoryCalloc(1, tLen);
void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, metaBuff, len); tEncoderInit(&encoder, metaBuff, len);
code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
@ -374,13 +375,13 @@ end:
} }
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) {
int32_t code = 0; int32_t code = 0;
STqOffsetVal reqOffset = {0}; STqOffsetVal reqOffset = {0};
tOffsetCopy(&reqOffset, &pRequest->reqOffset); tOffsetCopy(&reqOffset, &pRequest->reqOffset);
// reset the offset if needed // reset the offset if needed
if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type)) { if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type)) {
bool blockReturned = false; bool blockReturned = false;
code = extractResetOffsetVal(&reqOffset, pTq, pHandle, pRequest, pMsg, &blockReturned); code = extractResetOffsetVal(&reqOffset, pTq, pHandle, pRequest, pMsg, &blockReturned);
if (code != 0) { if (code != 0) {
goto END; goto END;
@ -392,7 +393,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
} }
} else if (reqOffset.type == 0) { // use the consumer specified offset } else if (reqOffset.type == 0) { // use the consumer specified offset
uError("req offset type is 0"); uError("req offset type is 0");
code = TSDB_CODE_TMQ_INVALID_MSG; code = TSDB_CODE_TMQ_INVALID_MSG;
goto END; goto END;
} }
@ -416,8 +417,8 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int
pMsgHead->walever = ever; pMsgHead->walever = ever;
} }
int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
int32_t vgId) { const SMqBatchMetaRsp* pRsp, int32_t vgId) {
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code); tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code);
@ -444,8 +445,8 @@ int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SM
SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0};
tmsgSendRsp(&resp); tmsgSendRsp(&resp);
tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", vgId, tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d",
pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type); vgId, pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type);
return 0; return 0;
} }
@ -527,6 +528,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp,
} }
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) { int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) {
int32_t code;
SDecoder* pCoder = &(SDecoder){0}; SDecoder* pCoder = &(SDecoder){0};
SDeleteRes* pRes = &(SDeleteRes){0}; SDeleteRes* pRes = &(SDeleteRes){0};
@ -570,11 +572,11 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
taosArrayDestroy(pRes->uidList); taosArrayDestroy(pRes->uidList);
if (type == 0) { if (type == 0) {
*pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); code = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0, pRefBlock);
if (*pRefBlock == NULL) { if (code) {
blockDataCleanup(pDelBlock); blockDataCleanup(pDelBlock);
taosMemoryFree(pDelBlock); taosMemoryFree(pDelBlock);
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK; ((SStreamRefDataBlock*)(*pRefBlock))->type = STREAM_INPUT__REF_DATA_BLOCK;
@ -614,7 +616,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
continue; continue;
} }
STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL) { if (ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId); tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId);

224
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1307,186 +1307,6 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
int nCols, int16_t *slotIds); int nCols, int16_t *slotIds);
#ifdef BUILD_NO_CALL
static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t ltype) {
SLastCol *pLastCol = NULL;
char *err = NULL;
size_t vlen = 0;
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
size_t klen = ROCKS_KEY_LEN;
char *value = NULL;
value = rocksdb_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, (char *)key, klen, &vlen, &err);
if (NULL != err) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, err);
rocksdb_free(err);
}
pLastCol = tsdbCacheDeserialize(value);
return pLastCol;
}
int32_t tsdbCacheGetSlow(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
rocksdb_writebatch_t *wb = NULL;
int32_t code = 0;
SArray *pCidList = pr->pCidList;
int num_keys = TARRAY_SIZE(pCidList);
char **keys_list = taosMemoryMalloc(num_keys * sizeof(char *));
size_t *keys_list_sizes = taosMemoryMalloc(num_keys * sizeof(size_t));
char *key_list = taosMemoryMalloc(num_keys * ROCKS_KEY_LEN);
for (int i = 0; i < num_keys; ++i) {
int16_t cid = *(int16_t *)taosArrayGet(pCidList, i);
memcpy(key_list + i * ROCKS_KEY_LEN, &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid}, ROCKS_KEY_LEN);
keys_list[i] = key_list + i * ROCKS_KEY_LEN;
keys_list_sizes[i] = ROCKS_KEY_LEN;
}
char **values_list = taosMemoryCalloc(num_keys, sizeof(char *));
size_t *values_list_sizes = taosMemoryCalloc(num_keys, sizeof(size_t));
char **errs = taosMemoryMalloc(num_keys * sizeof(char *));
rocksdb_multi_get(pTsdb->rCache.db, pTsdb->rCache.readoptions, num_keys, (const char *const *)keys_list,
keys_list_sizes, values_list, values_list_sizes, errs);
for (int i = 0; i < num_keys; ++i) {
if (errs[i]) {
rocksdb_free(errs[i]);
}
}
taosMemoryFree(key_list);
taosMemoryFree(keys_list);
taosMemoryFree(keys_list_sizes);
taosMemoryFree(errs);
for (int i = 0; i < num_keys; ++i) {
bool freeCol = true;
SArray *pTmpColArray = NULL;
SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i]);
int16_t cid = *(int16_t *)taosArrayGet(pCidList, i);
SLastCol noneCol = {.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[pr->pSlotIds[i]].type)};
if (pLastCol) {
reallocVarData(&pLastCol->colVal);
} else {
taosThreadMutexLock(&pTsdb->rCache.rMutex);
pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype);
if (!pLastCol) {
// recalc: load from tsdb
int16_t aCols[1] = {cid};
int16_t slotIds[1] = {pr->pSlotIds[i]};
pTmpColArray = NULL;
if (ltype) {
mergeLastCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds);
} else {
mergeLastRowCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds);
}
if (pTmpColArray && TARRAY_SIZE(pTmpColArray) >= 1) {
pLastCol = taosArrayGet(pTmpColArray, 0);
freeCol = false;
}
// still null, then make up a none col value
if (!pLastCol) {
pLastCol = &noneCol;
freeCol = false;
}
// store result back to rocks cache
wb = pTsdb->rCache.rwritebatch;
char *value = NULL;
size_t vlen = 0;
tsdbCacheSerialize(pLastCol, &value, &vlen);
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = pLastCol->colVal.cid};
size_t klen = ROCKS_KEY_LEN;
rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen);
taosMemoryFree(value);
} else {
reallocVarData(&pLastCol->colVal);
}
if (wb) {
rocksMayWrite(pTsdb, false, true, false);
}
taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
}
taosArrayPush(pLastArray, pLastCol);
taosArrayDestroy(pTmpColArray);
if (freeCol) {
taosMemoryFree(pLastCol);
}
}
taosMemoryFree(values_list);
taosMemoryFree(values_list_sizes);
return code;
}
static SLastCol *tsdbCacheLoadCol(STsdb *pTsdb, SCacheRowsReader *pr, int16_t slotid, tb_uid_t uid, int16_t cid,
int8_t ltype) {
SLastCol *pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype);
if (!pLastCol) {
rocksdb_writebatch_t *wb = NULL;
taosThreadMutexLock(&pTsdb->rCache.rMutex);
pLastCol = tsdbCacheLookup(pTsdb, uid, cid, ltype);
if (!pLastCol) {
// recalc: load from tsdb
int16_t aCols[1] = {cid};
int16_t slotIds[1] = {slotid};
SArray *pTmpColArray = NULL;
if (ltype) {
mergeLastCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds);
} else {
mergeLastRowCid(uid, pTsdb, &pTmpColArray, pr, aCols, 1, slotIds);
}
if (pTmpColArray && TARRAY_SIZE(pTmpColArray) >= 1) {
pLastCol = taosArrayGet(pTmpColArray, 0);
}
// still null, then make up a none col value
SLastCol noneCol = {.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[slotid].type)};
if (!pLastCol) {
pLastCol = &noneCol;
}
// store result back to rocks cache
wb = pTsdb->rCache.rwritebatch;
char *value = NULL;
size_t vlen = 0;
tsdbCacheSerialize(pLastCol, &value, &vlen);
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = pLastCol->colVal.cid};
size_t klen = ROCKS_KEY_LEN;
rocksdb_writebatch_put(wb, (char *)key, klen, value, vlen);
taosMemoryFree(value);
SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
*pTmpLastCol = *pLastCol;
pLastCol = pTmpLastCol;
taosArrayDestroy(pTmpColArray);
}
if (wb) {
rocksMayWrite(pTsdb, false, true, false);
}
taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
}
return pLastCol;
}
#endif
static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols, static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SArray *remainCols,
SCacheRowsReader *pr, int8_t ltype) { SCacheRowsReader *pr, int8_t ltype) {
@ -2579,7 +2399,7 @@ typedef struct {
static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid,
tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) { tb_uid_t uid, SCacheRowsReader *pr, int64_t lastTs, int16_t *aCols, int nCols) {
int32_t code = 0; int32_t code = 0;
pr->pLDataIterArray = destroySttBlockReader(pr->pLDataIterArray, NULL); destroySttBlockReader(pr->pLDataIterArray, NULL);
pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); pr->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
SMergeTreeConf conf = { SMergeTreeConf conf = {
@ -3212,7 +3032,10 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs
pIter->pMemDelData = NULL; pIter->pMemDelData = NULL;
loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer); code = loadMemTombData(&pIter->pMemDelData, pMem, pIMem, pr->info.verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
goto _err;
}
pIter->idx = (SBlockIdx){.suid = suid, .uid = uid}; pIter->idx = (SBlockIdx){.suid = suid, .uid = uid};
@ -3450,8 +3273,9 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
taosArrayPush(aColArray, &aCols[i]); taosArrayPush(aColArray, &aCols[i]);
} }
TSKEY lastRowTs = TSKEY_MAX; STsdbRowKey lastRowKey = {.key.ts = TSKEY_MAX};
// inverse iterator
CacheNextRowIter iter = {0}; CacheNextRowIter iter = {0};
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr);
@ -3475,10 +3299,11 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
} }
// int16_t nCol = pTSchema->numOfCols; // int16_t nCol = pTSchema->numOfCols;
TSKEY rowTs = TSDBROW_TS(pRow); STsdbRowKey rowKey = {0};
tsdbRowGetKey(pRow, &rowKey);
if (lastRowTs == TSKEY_MAX) { if (lastRowKey.key.ts == TSKEY_MAX) { // first time
lastRowTs = rowTs; lastRowKey = rowKey;
for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
if (iCol >= nLastCol) { if (iCol >= nLastCol) {
@ -3498,13 +3323,13 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
if (slotIds[iCol] == 0) { if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowTs})); *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts}));
taosArraySet(pColArray, 0, &(SLastCol){.rowKey.ts = rowTs, .colVal = *pColVal}); taosArraySet(pColArray, 0, &(SLastCol){.rowKey = rowKey.key, .colVal = *pColVal});
continue; continue;
} }
tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
*pCol = (SLastCol){.rowKey.ts = rowTs, .colVal = *pColVal}; *pCol = (SLastCol){.rowKey = rowKey.key, .colVal = *pColVal};
if (IS_VAR_DATA_TYPE(pColVal->value.type) /*&& pColVal->value.nData > 0*/) { if (IS_VAR_DATA_TYPE(pColVal->value.type) /*&& pColVal->value.nData > 0*/) {
if (pColVal->value.nData > 0) { if (pColVal->value.nData > 0) {
pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
@ -3551,10 +3376,11 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
continue; continue;
} }
SColVal *tColVal = &lastColVal->colVal; SColVal *tColVal = &lastColVal->colVal;
if (COL_VAL_IS_VALUE(tColVal)) continue;
tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) { if (COL_VAL_IS_VALUE(pColVal)) {
SLastCol lastCol = {.rowKey.ts = rowTs, .colVal = *pColVal}; SLastCol lastCol = {.rowKey = rowKey.key, .colVal = *pColVal};
if (IS_VAR_DATA_TYPE(pColVal->value.type) /* && pColVal->value.nData > 0 */) { if (IS_VAR_DATA_TYPE(pColVal->value.type) /* && pColVal->value.nData > 0 */) {
SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol); SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol);
taosMemoryFree(pLastCol->colVal.value.pData); taosMemoryFree(pLastCol->colVal.value.pData);
@ -3577,7 +3403,7 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
if (aColIndex >= 0) { if (aColIndex >= 0) {
taosArrayRemove(aColArray, aColIndex); taosArrayRemove(aColArray, aColIndex);
} }
} else if (!COL_VAL_IS_VALUE(tColVal) && !COL_VAL_IS_VALUE(pColVal) && !setNoneCol) { } else if (!COL_VAL_IS_VALUE(pColVal) && !setNoneCol) {
noneCol = iCol; noneCol = iCol;
setNoneCol = true; setNoneCol = true;
} }
@ -3634,8 +3460,7 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
taosArrayPush(aColArray, &aCols[i]); taosArrayPush(aColArray, &aCols[i]);
} }
TSKEY lastRowTs = TSKEY_MAX; // inverse iterator
CacheNextRowIter iter = {0}; CacheNextRowIter iter = {0};
nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr); nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->info.suid, pr->pLDataIterArray, pr->pReadSnap, pr->lastTs, pr);
@ -3659,9 +3484,8 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
} }
// int16_t nCol = pTSchema->numOfCols; // int16_t nCol = pTSchema->numOfCols;
TSKEY rowTs = TSDBROW_TS(pRow); STsdbRowKey rowKey = {0};
tsdbRowGetKey(pRow, &rowKey);
lastRowTs = rowTs;
for (int16_t iCol = noneCol; iCol < nCols; ++iCol) { for (int16_t iCol = noneCol; iCol < nCols; ++iCol) {
if (iCol >= nLastCol) { if (iCol >= nLastCol) {
@ -3677,13 +3501,13 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
if (slotIds[iCol] == 0) { if (slotIds[iCol] == 0) {
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowTs})); *pColVal = COL_VAL_VALUE(pTColumn->colId, ((SValue){.type = pTColumn->type, .val = rowKey.key.ts}));
taosArraySet(pColArray, 0, &(SLastCol){.rowKey.ts = rowTs, .colVal = *pColVal}); taosArraySet(pColArray, 0, &(SLastCol){.rowKey = rowKey.key, .colVal = *pColVal});
continue; continue;
} }
tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal); tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
*pCol = (SLastCol){.rowKey.ts = rowTs, .colVal = *pColVal}; *pCol = (SLastCol){.rowKey = rowKey.key, .colVal = *pColVal};
if (IS_VAR_DATA_TYPE(pColVal->value.type) /*&& pColVal->value.nData > 0*/) { if (IS_VAR_DATA_TYPE(pColVal->value.type) /*&& pColVal->value.nData > 0*/) {
if (pColVal->value.nData > 0) { if (pColVal->value.nData > 0) {
pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData); pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);

2
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -203,7 +203,7 @@ int32_t tsdbReuseCacherowsReader(void* reader, void* pTableIdList, int32_t numOf
pReader->pTableList = pTableIdList; pReader->pTableList = pTableIdList;
pReader->numOfTables = numOfTables; pReader->numOfTables = numOfTables;
pReader->lastTs = INT64_MIN; pReader->lastTs = INT64_MIN;
pReader->pLDataIterArray = destroySttBlockReader(pReader->pLDataIterArray, NULL); destroySttBlockReader(pReader->pLDataIterArray, NULL);
pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES); pReader->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;

35
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -23,11 +23,12 @@
static void tLDataIterClose2(SLDataIter *pIter); static void tLDataIterClose2(SLDataIter *pIter);
// SLDataIter ================================================= // SLDataIter =================================================
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols) { int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) {
*pInfo = NULL;
SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo)); SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo));
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} }
pLoadInfo->blockData[0].sttBlockIndex = -1; pLoadInfo->blockData[0].sttBlockIndex = -1;
@ -37,26 +38,29 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data); int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data);
if (code) { if (code) {
terrno = code; taosMemoryFreeClear(pLoadInfo);
return code;
} }
code = tBlockDataCreate(&pLoadInfo->blockData[1].data); code = tBlockDataCreate(&pLoadInfo->blockData[1].data);
if (code) { if (code) {
terrno = code; taosMemoryFreeClear(pLoadInfo);
return code;
} }
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) { if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo); taosMemoryFreeClear(pLoadInfo);
return NULL; return code;
} }
pLoadInfo->pSchema = pSchema; pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList; pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols; pLoadInfo->numOfCols = numOfCols;
return pLoadInfo; *pInfo = pLoadInfo;
return code;
} }
static void freeItem(void* pValue) { static void freeItem(void* pValue) {
@ -66,9 +70,9 @@ static void freeItem(void* pValue) {
} }
} }
void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
return NULL; return;
} }
pLoadInfo->currentLoadBlockIndex = 1; pLoadInfo->currentLoadBlockIndex = 1;
@ -94,7 +98,6 @@ void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {
taosArrayDestroy(pLoadInfo->aSttBlk); taosArrayDestroy(pLoadInfo->aSttBlk);
taosMemoryFree(pLoadInfo); taosMemoryFree(pLoadInfo);
return NULL;
} }
void destroyLDataIter(SLDataIter *pIter) { void destroyLDataIter(SLDataIter *pIter) {
@ -103,9 +106,9 @@ void destroyLDataIter(SLDataIter *pIter) {
taosMemoryFree(pIter); taosMemoryFree(pIter);
} }
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) { void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {
if (pLDataIterArray == NULL) { if (pLDataIterArray == NULL) {
return NULL; return;
} }
int32_t numOfLevel = taosArrayGetSize(pLDataIterArray); int32_t numOfLevel = taosArrayGetSize(pLDataIterArray);
@ -132,7 +135,6 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
} }
taosArrayDestroy(pLDataIterArray); taosArrayDestroy(pLDataIterArray);
return NULL;
} }
// choose the unpinned slot to load next data block // choose the unpinned slot to load next data block
@ -914,9 +916,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
} }
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo);
if (pLoadInfo == NULL) { if (code != TSDB_CODE_SUCCESS) {
code = terrno;
goto _end; goto _end;
} }
} }

761
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

File diff suppressed because it is too large Load Diff

112
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -112,22 +112,28 @@ void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
taosArrayDestroy(pBuf->pData); taosArrayDestroy(pBuf->pData);
} }
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index) { int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {
*pInfo = NULL;
int32_t bucketIndex = index / pBuf->numPerBucket; int32_t bucketIndex = index / pBuf->numPerBucket;
char** pBucket = taosArrayGet(pBuf->pData, bucketIndex); char** pBucket = taosArrayGet(pBuf->pData, bucketIndex);
return (*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo); if (pBucket == NULL) {
} return TSDB_CODE_FAILED;
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) {
STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (p == NULL || *p == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return NULL;
} }
return *p; *pInfo = (STableBlockScanInfo*)((*pBucket) + (index % pBuf->numPerBucket) * sizeof(STableBlockScanInfo));
return TSDB_CODE_SUCCESS;
}
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {
*pInfo = *(STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));
if (pInfo == NULL) {
int32_t size = tSimpleHashGetSize(pTableMap);
tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
return TSDB_CODE_INVALID_PARA;
}
return TSDB_CODE_SUCCESS;
} }
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) { int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
@ -146,17 +152,17 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
} }
case TSDB_DATA_TYPE_INT:{ case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN; int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_SMALLINT:{ case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN; int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_TINYINT:{ case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN; int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes); (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break; break;
} }
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
@ -245,20 +251,27 @@ int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSH
initLastProcKey(pScanInfo, pReader); initLastProcKey(pScanInfo, pReader);
pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); int32_t code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid, tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,
pScanInfo->lastProcKey.ts, pReader->idStr); pScanInfo->lastProcKey.ts, pReader->idStr);
return TSDB_CODE_SUCCESS; return code;
} }
// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables) { STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {
int32_t code = 0;
*pHashObj = NULL;
// allocate buffer in order to load data blocks from file // allocate buffer in order to load data blocks from file
// todo use simple hash instead, optimize the memory consumption // todo use simple hash instead, optimize the memory consumption
SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (pTableMap == NULL) { if (pTableMap == NULL) {
return NULL; return terrno;
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
@ -267,7 +280,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (pUidList->tableUidList == NULL) { if (pUidList->tableUidList == NULL) {
tSimpleHashCleanup(pTableMap); tSimpleHashCleanup(pTableMap);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pUidList->currentIndex = 0; pUidList->currentIndex = 0;
@ -275,8 +288,16 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
for (int32_t j = 0; j < numOfTables; ++j) { for (int32_t j = 0; j < numOfTables; ++j) {
pUidList->tableUidList[j] = idList[j].uid; pUidList->tableUidList[j] = idList[j].uid;
STableBlockScanInfo* pScanInfo = getPosInBlockInfoBuf(pBuf, j); STableBlockScanInfo* pScanInfo = NULL;
initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader); code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);
if (code != TSDB_CODE_SUCCESS) {
break;
}
code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
if (code != TSDB_CODE_SUCCESS) {
break;
}
} }
taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
@ -286,7 +307,8 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf
(sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,
pTsdbReader->idStr); pTsdbReader->idStr);
return pTableMap; *pHashObj = pTableMap;
return code;
} }
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
@ -391,11 +413,13 @@ void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray
pIter->pBrinBlockList = pList; pIter->pBrinBlockList = pList;
} }
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) { int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
*pRecord = NULL;
if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) { if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
pIter->blockIndex += 1; pIter->blockIndex += 1;
if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) { if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
return NULL; return TSDB_CODE_FAILED;
} }
pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex); pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
@ -404,7 +428,7 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block); int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code)); tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
return NULL; return TSDB_CODE_FAILED;
} }
pIter->recordIndex = -1; pIter->recordIndex = -1;
@ -412,7 +436,9 @@ SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter) {
pIter->recordIndex += 1; pIter->recordIndex += 1;
tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record); tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
return &pIter->record; *pRecord = &pIter->record;
return TSDB_CODE_SUCCESS;
} }
void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); } void clearBrinBlockIter(SBrinRecordIter* pIter) { tBrinBlockDestroy(&pIter->block); }
@ -670,7 +696,13 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
STombRecord record = {0}; STombRecord record = {0};
uint64_t uid = pReader->status.uidList.tableUidList[*j]; uint64_t uid = pReader->status.uidList.tableUidList[*j];
STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); STableBlockScanInfo* pScanInfo = NULL;
code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) { if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
} }
@ -702,7 +734,11 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
} }
uid = pReader->status.uidList.tableUidList[*j]; uid = pReader->status.uidList.tableUidList[*j];
pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (pScanInfo->pFileDelData == NULL) { if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData)); pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
} }
@ -806,9 +842,12 @@ int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileRead
return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false); return doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);
} }
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) { int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
if (*ppMemDelData == NULL) { if (*ppMemDelData == NULL) {
*ppMemDelData = taosArrayInit(4, sizeof(SDelData)); *ppMemDelData = taosArrayInit(4, sizeof(SDelData));
if (*ppMemDelData == NULL) {
return TSDB_CODE_SUCCESS;
}
} }
SArray* pMemDelData = *ppMemDelData; SArray* pMemDelData = *ppMemDelData;
@ -836,6 +875,8 @@ void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemT
p = p->pNext; p = p->pNext;
} }
} }
return TSDB_CODE_SUCCESS;
} }
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -960,6 +1001,7 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet)
int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf, int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
const char* pstr) { const char* pstr) {
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t code = 0;
// no data exists, go to end // no data exists, go to end
int32_t numOfLevels = pFileSet->lvlArr->size; int32_t numOfLevels = pFileSet->lvlArr->size;
@ -984,7 +1026,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
conf.file[0] = *pSttLevel->fobjArr->data[i]->f; conf.file[0] = *pSttLevel->fobjArr->data[i]->f;
const char* pName = pSttLevel->fobjArr->data[i]->fname; const char* pName = pSttLevel->fobjArr->data[i]->fname;
int32_t code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader); code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr); tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);
continue; continue;
@ -992,8 +1034,8 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
} }
if (pIter->pBlockLoadInfo == NULL) { if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);
if (pIter->pBlockLoadInfo == NULL) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr); tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue; continue;
} }
@ -1001,7 +1043,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
TStatisBlkArray* pStatisBlkArray = NULL; TStatisBlkArray* pStatisBlkArray = NULL;
int32_t code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray); code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr); tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
continue; continue;

41
source/dnode/vnode/src/tsdb/tsdbReadUtil.h
View File

@ -315,32 +315,31 @@ typedef struct SBrinRecordIter {
} SBrinRecordIter; } SBrinRecordIter;
int32_t uidComparFunc(const void* p1, const void* p2); int32_t uidComparFunc(const void* p1, const void* p2);
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id);
STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj);
SSHashObj* createDataBlockScanInfo(STsdbReader* pReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
STableUidList* pUidList, int32_t numOfTables); STsdbReader* pReader);
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap, void clearBlockScanInfo(STableBlockScanInfo* p);
STsdbReader* pReader); void destroyAllBlockScanInfo(SSHashObj* pTableMap);
void clearBlockScanInfo(STableBlockScanInfo* p); void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step);
void destroyAllBlockScanInfo(SSHashObj* pTableMap); void cleanupInfoForNextFileset(SSHashObj* pTableMap);
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables);
void cleanupInfoForNextFileset(SSHashObj* pTableMap); void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pRes);
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf);
void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index);
// brin records iterator // brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList); void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList);
SBrinRecord* getNextBrinRecord(SBrinRecordIter* pIter); int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord);
void clearBrinBlockIter(SBrinRecordIter* pIter); void clearBrinBlockIter(SBrinRecordIter* pIter);
// initialize block iterator API // initialize block iterator API
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList); int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList);
bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr); bool blockIteratorNext(SDataBlockIter* pBlockIter, const char* idStr);
// load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time) // load tomb data API (stt/mem only for one table each, tomb data from data files are load for all tables at one time)
void loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver); int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver);
int32_t loadDataFileTombDataForAll(STsdbReader* pReader); int32_t loadDataFileTombDataForAll(STsdbReader* pReader);
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo); int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo);
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo, int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
@ -358,10 +357,10 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2);
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc); int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc);
void clearRowKey(SRowKey* pKey); void clearRowKey(SRowKey* pKey);
bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp); bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp);
void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk); int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool hasPk);
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree); void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree);
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk); void cleanupDataBlockIterator(SDataBlockIter* pIter, bool hasPk);
typedef struct { typedef struct {
SArray* pTombData; SArray* pTombData;

35
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -37,8 +37,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
int8_t cmprAlg; int8_t cmprAlg;
int32_t szPage; int32_t szPage;
SBuffer buffers[10]; SBuffer buffers[10];
int32_t encryptAlgorithm; int32_t encryptAlgorithm;
char* encryptKey; char *encryptKey;
// reader // reader
SArray *aBlockIdx; SArray *aBlockIdx;
SMapData mDataBlk[1]; SMapData mDataBlk[1];
@ -139,8 +139,9 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
if (ctx->brinBlock->numOfRecords >= ctx->maxRow) { if (ctx->brinBlock->numOfRecords >= ctx->maxRow) {
SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN}; SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size, code =
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey); tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
ctx->brinBlkArray, ctx->buffers, &range, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
@ -157,8 +158,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
&fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey); &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm, code = tsdbFileWriteHeadFooter(ctx->fd, &fset->farr[TSDB_FTYPE_HEAD]->f->size, ctx->footer, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey); code = tsdbFsyncFile(ctx->fd, ctx->encryptAlgorithm, ctx->encryptKey);
@ -258,7 +259,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
struct { struct {
int32_t szPage; int32_t szPage;
int32_t encryptAlgorithm; int32_t encryptAlgorithm;
char* encryptKey; char *encryptKey;
// writer // writer
STsdbFD *fd; STsdbFD *fd;
TSttBlkArray sttBlkArray[1]; TSttBlkArray sttBlkArray[1];
@ -290,7 +291,7 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
} }
code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm, code = tsdbFileWriteSttBlk(ctx->fd, ctx->sttBlkArray, ctx->footer->sttBlkPtr, &fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit1); TSDB_CHECK_CODE(code, lino, _exit1);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey); code = tsdbFileWriteSttFooter(ctx->fd, ctx->footer, &fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
@ -442,7 +443,7 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
uint8_t hdr[TSDB_FHDR_SIZE] = {0}; uint8_t hdr[TSDB_FHDR_SIZE] = {0};
int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; int32_t encryptAlgorithm = tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
char* encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey; char *encryptKey = tsdb->pVnode->config.tsdbCfg.encryptKey;
code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey); code = tsdbWriteFile(fd[0], 0, hdr, TSDB_FHDR_SIZE, encryptAlgorithm, encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -467,8 +468,8 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
int64_t minKey; int64_t minKey;
int64_t maxKey; int64_t maxKey;
SBuffer buffers[10]; SBuffer buffers[10];
int32_t encryptAlgorithm; int32_t encryptAlgorithm;
char* encryptKey; char *encryptKey;
// reader // reader
SArray *aDelData; SArray *aDelData;
// writer // writer
@ -538,20 +539,20 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
if (ctx->fd != NULL) { if (ctx->fd != NULL) {
if (ctx->toStt) { if (ctx->toStt) {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size, code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->sttFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey); ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, code =
ctx->encryptKey); tsdbFileWriteSttFooter(ctx->fd, ctx->sttFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size, code = tsdbFileWriteTombBlk(ctx->fd, ctx->tombBlkArray, ctx->tombFooter->tombBlkPtr, &ctx->fobj->f->size,
ctx->encryptAlgorithm, ctx->encryptKey); ctx->encryptAlgorithm, ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm, code = tsdbFileWriteTombFooter(ctx->fd, ctx->tombFooter, &ctx->fobj->f->size, ctx->encryptAlgorithm,
ctx->encryptKey); ctx->encryptKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }

258
source/dnode/vnode/src/tsdb/tsdbUtil2.c
View File

@ -17,17 +17,24 @@
// SDelBlock ---------- // SDelBlock ----------
int32_t tTombBlockInit(STombBlock *tombBlock) { int32_t tTombBlockInit(STombBlock *tombBlock) {
int32_t code;
tombBlock->numOfRecords = 0; tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
tBufferInit(&tombBlock->buffers[i]); TAOS_CHECK_GOTO(tBufferInit(&tombBlock->buffers[i]), NULL, _exit);
} }
return 0;
_exit:
if (code) {
TAOS_UNUSED(tTombBlockDestroy(tombBlock));
}
return code;
} }
int32_t tTombBlockDestroy(STombBlock *tombBlock) { int32_t tTombBlockDestroy(STombBlock *tombBlock) {
tombBlock->numOfRecords = 0; tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
tBufferDestroy(&tombBlock->buffers[i]); TAOS_UNUSED(tBufferDestroy(&tombBlock->buffers[i]));
} }
return 0; return 0;
} }
@ -35,15 +42,14 @@ int32_t tTombBlockDestroy(STombBlock *tombBlock) {
int32_t tTombBlockClear(STombBlock *tombBlock) { int32_t tTombBlockClear(STombBlock *tombBlock) {
tombBlock->numOfRecords = 0; tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
tBufferClear(&tombBlock->buffers[i]); TAOS_UNUSED(tBufferClear(&tombBlock->buffers[i]));
} }
return 0; return 0;
} }
int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record) { int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record) {
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
int32_t code = tBufferPutI64(&tombBlock->buffers[i], record->data[i]); TAOS_CHECK_RETURN(tBufferPutI64(&tombBlock->buffers[i], record->data[i]));
if (code) return code;
} }
tombBlock->numOfRecords++; tombBlock->numOfRecords++;
return 0; return 0;
@ -56,8 +62,7 @@ int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) {
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * idx, &tombBlock->buffers[i]); SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * idx, &tombBlock->buffers[i]);
int32_t code = tBufferGetI64(&br, &record->data[i]); TAOS_CHECK_RETURN(tBufferGetI64(&br, &record->data[i]));
if (code) return code;
} }
return 0; return 0;
} }
@ -74,27 +79,34 @@ int32_t tTombRecordCompare(const STombRecord *r1, const STombRecord *r2) {
// STbStatisBlock ---------- // STbStatisBlock ----------
int32_t tStatisBlockInit(STbStatisBlock *statisBlock) { int32_t tStatisBlockInit(STbStatisBlock *statisBlock) {
int32_t code;
statisBlock->numOfPKs = 0; statisBlock->numOfPKs = 0;
statisBlock->numOfRecords = 0; statisBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
tBufferInit(&statisBlock->buffers[i]); TAOS_CHECK_GOTO(tBufferInit(&statisBlock->buffers[i]), NULL, _exit);
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnInit(&statisBlock->firstKeyPKs[i]); TAOS_CHECK_GOTO(tValueColumnInit(&statisBlock->firstKeyPKs[i]), NULL, _exit);
tValueColumnInit(&statisBlock->lastKeyPKs[i]); TAOS_CHECK_GOTO(tValueColumnInit(&statisBlock->lastKeyPKs[i]), NULL, _exit);
} }
return 0;
_exit:
if (code) {
TAOS_UNUSED(tStatisBlockDestroy(statisBlock));
}
return code;
} }
int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock) { int32_t tStatisBlockDestroy(STbStatisBlock *statisBlock) {
statisBlock->numOfPKs = 0; statisBlock->numOfPKs = 0;
statisBlock->numOfRecords = 0; statisBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
tBufferDestroy(&statisBlock->buffers[i]); TAOS_UNUSED(tBufferDestroy(&statisBlock->buffers[i]));
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnDestroy(&statisBlock->firstKeyPKs[i]); TAOS_UNUSED(tValueColumnDestroy(&statisBlock->firstKeyPKs[i]));
tValueColumnDestroy(&statisBlock->lastKeyPKs[i]); TAOS_UNUSED(tValueColumnDestroy(&statisBlock->lastKeyPKs[i]));
} }
return 0; return 0;
} }
@ -103,17 +115,16 @@ int32_t tStatisBlockClear(STbStatisBlock *statisBlock) {
statisBlock->numOfPKs = 0; statisBlock->numOfPKs = 0;
statisBlock->numOfRecords = 0; statisBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
tBufferClear(&statisBlock->buffers[i]); TAOS_UNUSED(tBufferClear(&statisBlock->buffers[i]));
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnClear(&statisBlock->firstKeyPKs[i]); TAOS_UNUSED(tValueColumnClear(&statisBlock->firstKeyPKs[i]));
tValueColumnClear(&statisBlock->lastKeyPKs[i]); TAOS_UNUSED(tValueColumnClear(&statisBlock->lastKeyPKs[i]));
} }
return 0; return 0;
} }
static int32_t tStatisBlockAppend(STbStatisBlock *block, SRowInfo *row) { static int32_t tStatisBlockAppend(STbStatisBlock *block, SRowInfo *row) {
int32_t code;
STsdbRowKey key; STsdbRowKey key;
tsdbRowGetKey(&row->row, &key); tsdbRowGetKey(&row->row, &key);
@ -129,14 +140,14 @@ static int32_t tStatisBlockAppend(STbStatisBlock *block, SRowInfo *row) {
} }
} }
if ((code = tBufferPutI64(&block->suids, row->suid))) return code; TAOS_CHECK_RETURN(tBufferPutI64(&block->suids, row->suid));
if ((code = tBufferPutI64(&block->uids, row->uid))) return code; TAOS_CHECK_RETURN(tBufferPutI64(&block->uids, row->uid));
if ((code = tBufferPutI64(&block->firstKeyTimestamps, key.key.ts))) return code; TAOS_CHECK_RETURN(tBufferPutI64(&block->firstKeyTimestamps, key.key.ts));
if ((code = tBufferPutI64(&block->lastKeyTimestamps, key.key.ts))) return code; TAOS_CHECK_RETURN(tBufferPutI64(&block->lastKeyTimestamps, key.key.ts));
if ((code = tBufferPutI64(&block->counts, 1))) return code; TAOS_CHECK_RETURN(tBufferPutI64(&block->counts, 1));
for (int32_t i = 0; i < block->numOfPKs; ++i) { for (int32_t i = 0; i < block->numOfPKs; ++i) {
if ((code = tValueColumnAppend(block->firstKeyPKs + i, key.key.pks + i))) return code; TAOS_CHECK_RETURN(tValueColumnAppend(block->firstKeyPKs + i, key.key.pks + i));
if ((code = tValueColumnAppend(block->lastKeyPKs + i, key.key.pks + i))) return code; TAOS_CHECK_RETURN(tValueColumnAppend(block->lastKeyPKs + i, key.key.pks + i));
} }
block->numOfRecords++; block->numOfRecords++;
@ -147,9 +158,8 @@ static int32_t tStatisBlockUpdate(STbStatisBlock *block, SRowInfo *row) {
STbStatisRecord record; STbStatisRecord record;
STsdbRowKey key; STsdbRowKey key;
int32_t c; int32_t c;
int32_t code;
tStatisBlockGet(block, block->numOfRecords - 1, &record); TAOS_CHECK_RETURN(tStatisBlockGet(block, block->numOfRecords - 1, &record));
tsdbRowGetKey(&row->row, &key); tsdbRowGetKey(&row->row, &key);
c = tRowKeyCompare(&record.lastKey, &key.key); c = tRowKeyCompare(&record.lastKey, &key.key);
@ -157,21 +167,18 @@ static int32_t tStatisBlockUpdate(STbStatisBlock *block, SRowInfo *row) {
return 0; return 0;
} else if (c < 0) { } else if (c < 0) {
// last ts // last ts
code = tBufferPutAt(&block->lastKeyTimestamps, (block->numOfRecords - 1) * sizeof(record.lastKey.ts), &key.key.ts, TAOS_CHECK_RETURN(tBufferPutAt(&block->lastKeyTimestamps, (block->numOfRecords - 1) * sizeof(record.lastKey.ts),
sizeof(key.key.ts)); &key.key.ts, sizeof(key.key.ts)));
if (code) return code;
// last primary keys // last primary keys
for (int i = 0; i < block->numOfPKs; i++) { for (int i = 0; i < block->numOfPKs; i++) {
code = tValueColumnUpdate(&block->lastKeyPKs[i], block->numOfRecords - 1, &key.key.pks[i]); TAOS_CHECK_RETURN(tValueColumnUpdate(&block->lastKeyPKs[i], block->numOfRecords - 1, &key.key.pks[i]));
if (code) return code;
} }
// count // count
record.count++; record.count++;
code = tBufferPutAt(&block->counts, (block->numOfRecords - 1) * sizeof(record.count), &record.count, TAOS_CHECK_RETURN(tBufferPutAt(&block->counts, (block->numOfRecords - 1) * sizeof(record.count), &record.count,
sizeof(record.count)); sizeof(record.count)));
if (code) return code;
} else { } else {
ASSERT(0); ASSERT(0);
} }
@ -183,8 +190,7 @@ int32_t tStatisBlockPut(STbStatisBlock *block, SRowInfo *row, int32_t maxRecords
if (block->numOfRecords > 0) { if (block->numOfRecords > 0) {
int64_t lastUid; int64_t lastUid;
SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * (block->numOfRecords - 1), &block->uids); SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * (block->numOfRecords - 1), &block->uids);
int32_t code = tBufferGetI64(&br, &lastUid); TAOS_CHECK_RETURN(tBufferGetI64(&br, &lastUid));
if (code) return code;
if (lastUid == row->uid) { if (lastUid == row->uid) {
return tStatisBlockUpdate(block, row); return tStatisBlockUpdate(block, row);
@ -196,7 +202,6 @@ int32_t tStatisBlockPut(STbStatisBlock *block, SRowInfo *row, int32_t maxRecords
} }
int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record) { int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecord *record) {
int32_t code;
SBufferReader reader; SBufferReader reader;
if (idx < 0 || idx >= statisBlock->numOfRecords) { if (idx < 0 || idx >= statisBlock->numOfRecords) {
@ -204,36 +209,29 @@ int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecor
} }
reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->suid), &statisBlock->suids); reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->suid), &statisBlock->suids);
code = tBufferGetI64(&reader, &record->suid); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->suid));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->uid), &statisBlock->uids); reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->uid), &statisBlock->uids);
code = tBufferGetI64(&reader, &record->uid); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->uid));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->firstKey.ts), &statisBlock->firstKeyTimestamps); reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->firstKey.ts), &statisBlock->firstKeyTimestamps);
code = tBufferGetI64(&reader, &record->firstKey.ts); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->firstKey.ts));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->lastKey.ts), &statisBlock->lastKeyTimestamps); reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->lastKey.ts), &statisBlock->lastKeyTimestamps);
code = tBufferGetI64(&reader, &record->lastKey.ts); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->lastKey.ts));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->count), &statisBlock->counts); reader = BUFFER_READER_INITIALIZER(idx * sizeof(record->count), &statisBlock->counts);
code = tBufferGetI64(&reader, &record->count); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->count));
if (code) return code;
// primary keys // primary keys
for (record->firstKey.numOfPKs = 0; record->firstKey.numOfPKs < statisBlock->numOfPKs; record->firstKey.numOfPKs++) { for (record->firstKey.numOfPKs = 0; record->firstKey.numOfPKs < statisBlock->numOfPKs; record->firstKey.numOfPKs++) {
code = tValueColumnGet(&statisBlock->firstKeyPKs[record->firstKey.numOfPKs], idx, TAOS_CHECK_RETURN(tValueColumnGet(&statisBlock->firstKeyPKs[record->firstKey.numOfPKs], idx,
&record->firstKey.pks[record->firstKey.numOfPKs]); &record->firstKey.pks[record->firstKey.numOfPKs]));
if (code) return code;
} }
for (record->lastKey.numOfPKs = 0; record->lastKey.numOfPKs < statisBlock->numOfPKs; record->lastKey.numOfPKs++) { for (record->lastKey.numOfPKs = 0; record->lastKey.numOfPKs < statisBlock->numOfPKs; record->lastKey.numOfPKs++) {
code = tValueColumnGet(&statisBlock->lastKeyPKs[record->lastKey.numOfPKs], idx, TAOS_CHECK_RETURN(tValueColumnGet(&statisBlock->lastKeyPKs[record->lastKey.numOfPKs], idx,
&record->lastKey.pks[record->lastKey.numOfPKs]); &record->lastKey.pks[record->lastKey.numOfPKs]));
if (code) return code;
} }
return 0; return 0;
@ -241,27 +239,34 @@ int32_t tStatisBlockGet(STbStatisBlock *statisBlock, int32_t idx, STbStatisRecor
// SBrinRecord ---------- // SBrinRecord ----------
int32_t tBrinBlockInit(SBrinBlock *brinBlock) { int32_t tBrinBlockInit(SBrinBlock *brinBlock) {
int32_t code;
brinBlock->numOfPKs = 0; brinBlock->numOfPKs = 0;
brinBlock->numOfRecords = 0; brinBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) {
tBufferInit(&brinBlock->buffers[i]); TAOS_CHECK_GOTO(tBufferInit(&brinBlock->buffers[i]), NULL, _exit);
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnInit(&brinBlock->firstKeyPKs[i]); TAOS_CHECK_GOTO(tValueColumnInit(&brinBlock->firstKeyPKs[i]), NULL, _exit);
tValueColumnInit(&brinBlock->lastKeyPKs[i]); TAOS_CHECK_GOTO(tValueColumnInit(&brinBlock->lastKeyPKs[i]), NULL, _exit);
} }
return 0;
_exit:
if (code) {
TAOS_UNUSED(tBrinBlockDestroy(brinBlock));
}
return code;
} }
int32_t tBrinBlockDestroy(SBrinBlock *brinBlock) { int32_t tBrinBlockDestroy(SBrinBlock *brinBlock) {
brinBlock->numOfPKs = 0; brinBlock->numOfPKs = 0;
brinBlock->numOfRecords = 0; brinBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) {
tBufferDestroy(&brinBlock->buffers[i]); TAOS_UNUSED(tBufferDestroy(&brinBlock->buffers[i]));
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnDestroy(&brinBlock->firstKeyPKs[i]); TAOS_UNUSED(tValueColumnDestroy(&brinBlock->firstKeyPKs[i]));
tValueColumnDestroy(&brinBlock->lastKeyPKs[i]); TAOS_UNUSED(tValueColumnDestroy(&brinBlock->lastKeyPKs[i]));
} }
return 0; return 0;
} }
@ -270,18 +275,16 @@ int32_t tBrinBlockClear(SBrinBlock *brinBlock) {
brinBlock->numOfPKs = 0; brinBlock->numOfPKs = 0;
brinBlock->numOfRecords = 0; brinBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) { for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) {
tBufferClear(&brinBlock->buffers[i]); TAOS_UNUSED(tBufferClear(&brinBlock->buffers[i]));
} }
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) { for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
tValueColumnClear(&brinBlock->firstKeyPKs[i]); TAOS_UNUSED(tValueColumnClear(&brinBlock->firstKeyPKs[i]));
tValueColumnClear(&brinBlock->lastKeyPKs[i]); TAOS_UNUSED(tValueColumnClear(&brinBlock->lastKeyPKs[i]));
} }
return 0; return 0;
} }
int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) { int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) {
int32_t code;
ASSERT(record->firstKey.key.numOfPKs == record->lastKey.key.numOfPKs); ASSERT(record->firstKey.key.numOfPKs == record->lastKey.key.numOfPKs);
if (brinBlock->numOfRecords == 0) { // the first row if (brinBlock->numOfRecords == 0) { // the first row
@ -298,60 +301,29 @@ int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) {
} }
} }
code = tBufferPutI64(&brinBlock->suids, record->suid); TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->suids, record->suid));
if (code) return code; TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->uids, record->uid));
TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->firstKeyTimestamps, record->firstKey.key.ts));
code = tBufferPutI64(&brinBlock->uids, record->uid); TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->firstKeyVersions, record->firstKey.version));
if (code) return code; TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->lastKeyTimestamps, record->lastKey.key.ts));
TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->lastKeyVersions, record->lastKey.version));
code = tBufferPutI64(&brinBlock->firstKeyTimestamps, record->firstKey.key.ts); TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->minVers, record->minVer));
if (code) return code; TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->maxVers, record->maxVer));
TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->blockOffsets, record->blockOffset));
code = tBufferPutI64(&brinBlock->firstKeyVersions, record->firstKey.version); TAOS_CHECK_RETURN(tBufferPutI64(&brinBlock->smaOffsets, record->smaOffset));
if (code) return code; TAOS_CHECK_RETURN(tBufferPutI32(&brinBlock->blockSizes, record->blockSize));
TAOS_CHECK_RETURN(tBufferPutI32(&brinBlock->blockKeySizes, record->blockKeySize));
code = tBufferPutI64(&brinBlock->lastKeyTimestamps, record->lastKey.key.ts); TAOS_CHECK_RETURN(tBufferPutI32(&brinBlock->smaSizes, record->smaSize));
if (code) return code; TAOS_CHECK_RETURN(tBufferPutI32(&brinBlock->numRows, record->numRow));
TAOS_CHECK_RETURN(tBufferPutI32(&brinBlock->counts, record->count));
code = tBufferPutI64(&brinBlock->lastKeyVersions, record->lastKey.version);
if (code) return code;
code = tBufferPutI64(&brinBlock->minVers, record->minVer);
if (code) return code;
code = tBufferPutI64(&brinBlock->maxVers, record->maxVer);
if (code) return code;
code = tBufferPutI64(&brinBlock->blockOffsets, record->blockOffset);
if (code) return code;
code = tBufferPutI64(&brinBlock->smaOffsets, record->smaOffset);
if (code) return code;
code = tBufferPutI32(&brinBlock->blockSizes, record->blockSize);
if (code) return code;
code = tBufferPutI32(&brinBlock->blockKeySizes, record->blockKeySize);
if (code) return code;
code = tBufferPutI32(&brinBlock->smaSizes, record->smaSize);
if (code) return code;
code = tBufferPutI32(&brinBlock->numRows, record->numRow);
if (code) return code;
code = tBufferPutI32(&brinBlock->counts, record->count);
if (code) return code;
if (brinBlock->numOfPKs > 0) { if (brinBlock->numOfPKs > 0) {
for (int32_t i = 0; i < brinBlock->numOfPKs; ++i) { for (int32_t i = 0; i < brinBlock->numOfPKs; ++i) {
code = tValueColumnAppend(&brinBlock->firstKeyPKs[i], &record->firstKey.key.pks[i]); TAOS_CHECK_RETURN(tValueColumnAppend(&brinBlock->firstKeyPKs[i], &record->firstKey.key.pks[i]));
if (code) return code;
} }
for (int32_t i = 0; i < brinBlock->numOfPKs; ++i) { for (int32_t i = 0; i < brinBlock->numOfPKs; ++i) {
code = tValueColumnAppend(&brinBlock->lastKeyPKs[i], &record->lastKey.key.pks[i]); TAOS_CHECK_RETURN(tValueColumnAppend(&brinBlock->lastKeyPKs[i], &record->lastKey.key.pks[i]));
if (code) return code;
} }
} }
@ -361,7 +333,6 @@ int32_t tBrinBlockPut(SBrinBlock *brinBlock, const SBrinRecord *record) {
} }
int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record) { int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record) {
int32_t code;
SBufferReader reader; SBufferReader reader;
if (idx < 0 || idx >= brinBlock->numOfRecords) { if (idx < 0 || idx >= brinBlock->numOfRecords) {
@ -369,78 +340,61 @@ int32_t tBrinBlockGet(SBrinBlock *brinBlock, int32_t idx, SBrinRecord *record) {
} }
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->suids); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->suids);
code = tBufferGetI64(&reader, &record->suid); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->suid));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->uids); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->uids);
code = tBufferGetI64(&reader, &record->uid); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->uid));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->firstKeyTimestamps); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->firstKeyTimestamps);
code = tBufferGetI64(&reader, &record->firstKey.key.ts); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->firstKey.key.ts));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->firstKeyVersions); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->firstKeyVersions);
code = tBufferGetI64(&reader, &record->firstKey.version); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->firstKey.version));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->lastKeyTimestamps); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->lastKeyTimestamps);
code = tBufferGetI64(&reader, &record->lastKey.key.ts); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->lastKey.key.ts));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->lastKeyVersions); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->lastKeyVersions);
code = tBufferGetI64(&reader, &record->lastKey.version); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->lastKey.version));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->minVers); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->minVers);
code = tBufferGetI64(&reader, &record->minVer); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->minVer));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->maxVers); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->maxVers);
code = tBufferGetI64(&reader, &record->maxVer); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->maxVer));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->blockOffsets); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->blockOffsets);
code = tBufferGetI64(&reader, &record->blockOffset); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->blockOffset));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->smaOffsets); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int64_t), &brinBlock->smaOffsets);
code = tBufferGetI64(&reader, &record->smaOffset); TAOS_CHECK_RETURN(tBufferGetI64(&reader, &record->smaOffset));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->blockSizes); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->blockSizes);
code = tBufferGetI32(&reader, &record->blockSize); TAOS_CHECK_RETURN(tBufferGetI32(&reader, &record->blockSize));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->blockKeySizes); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->blockKeySizes);
code = tBufferGetI32(&reader, &record->blockKeySize); TAOS_CHECK_RETURN(tBufferGetI32(&reader, &record->blockKeySize));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->smaSizes); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->smaSizes);
code = tBufferGetI32(&reader, &record->smaSize); TAOS_CHECK_RETURN(tBufferGetI32(&reader, &record->smaSize));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->numRows); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->numRows);
code = tBufferGetI32(&reader, &record->numRow); TAOS_CHECK_RETURN(tBufferGetI32(&reader, &record->numRow));
if (code) return code;
reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->counts); reader = BUFFER_READER_INITIALIZER(idx * sizeof(int32_t), &brinBlock->counts);
code = tBufferGetI32(&reader, &record->count); TAOS_CHECK_RETURN(tBufferGetI32(&reader, &record->count));
if (code) return code;
// primary keys // primary keys
for (record->firstKey.key.numOfPKs = 0; record->firstKey.key.numOfPKs < brinBlock->numOfPKs; for (record->firstKey.key.numOfPKs = 0; record->firstKey.key.numOfPKs < brinBlock->numOfPKs;
record->firstKey.key.numOfPKs++) { record->firstKey.key.numOfPKs++) {
code = tValueColumnGet(&brinBlock->firstKeyPKs[record->firstKey.key.numOfPKs], idx, TAOS_CHECK_RETURN(tValueColumnGet(&brinBlock->firstKeyPKs[record->firstKey.key.numOfPKs], idx,
&record->firstKey.key.pks[record->firstKey.key.numOfPKs]); &record->firstKey.key.pks[record->firstKey.key.numOfPKs]));
if (code) return code;
} }
for (record->lastKey.key.numOfPKs = 0; record->lastKey.key.numOfPKs < brinBlock->numOfPKs; for (record->lastKey.key.numOfPKs = 0; record->lastKey.key.numOfPKs < brinBlock->numOfPKs;
record->lastKey.key.numOfPKs++) { record->lastKey.key.numOfPKs++) {
code = tValueColumnGet(&brinBlock->lastKeyPKs[record->lastKey.key.numOfPKs], idx, TAOS_CHECK_RETURN(tValueColumnGet(&brinBlock->lastKeyPKs[record->lastKey.key.numOfPKs], idx,
&record->lastKey.key.pks[record->lastKey.key.numOfPKs]); &record->lastKey.key.pks[record->lastKey.key.numOfPKs]));
if (code) return code;
} }
return 0; return 0;

4
source/libs/audit/src/auditMain.c
View File

@ -32,8 +32,8 @@ char* tsAuditBatchUri = "/audit-batch";
int32_t auditInit(const SAuditCfg *pCfg) { int32_t auditInit(const SAuditCfg *pCfg) {
tsAudit.cfg = *pCfg; tsAudit.cfg = *pCfg;
tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *)); tsAudit.records = taosArrayInit(0, sizeof(SAuditRecord *));
taosThreadMutexInit(&tsAudit.lock, NULL); if(tsAudit.records == NULL) return TSDB_CODE_OUT_OF_MEMORY;
return 0; return taosThreadMutexInit(&tsAudit.lock, NULL);
} }
static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) { static FORCE_INLINE void auditDeleteRecord(SAuditRecord * record) {

2
source/libs/executor/inc/executorInt.h
View File

@ -929,7 +929,7 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, i
SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore,
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex); SStorageAPI* pApi, int32_t tsIndex);
void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, int32_t initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic); STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic);
void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins);
void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);

1
source/libs/executor/inc/streamexecutorInt.h
View File

@ -20,6 +20,7 @@ extern "C" {
#endif #endif
#include "executorInt.h" #include "executorInt.h"
#include "tutil.h"
void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type); void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo); bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);

17
source/libs/executor/src/dataDeleter.c
View File

@ -130,9 +130,11 @@ static int32_t getStatus(SDataDeleterHandle* pDeleter) {
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0); SDataDeleterBuf* pBuf;
if (NULL == pBuf) {
return TSDB_CODE_OUT_OF_MEMORY; int32_t code = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
} }
if (!allocBuf(pDeleter, pInput, pBuf)) { if (!allocBuf(pDeleter, pInput, pBuf)) {
@ -227,7 +229,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
} }
taosCloseQueue(pDeleter->pDataBlocks); taosCloseQueue(pDeleter->pDataBlocks);
taosThreadMutexDestroy(&pDeleter->mutex); taosThreadMutexDestroy(&pDeleter->mutex);
taosMemoryFree(pDeleter->pManager); taosMemoryFree(pDeleter->pManager);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -270,12 +272,11 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData
deleter->pParam = pParam; deleter->pParam = pParam;
deleter->status = DS_BUF_EMPTY; deleter->status = DS_BUF_EMPTY;
deleter->queryEnd = false; deleter->queryEnd = false;
deleter->pDataBlocks = taosOpenQueue(); code = taosOpenQueue(&deleter->pDataBlocks);
taosThreadMutexInit(&deleter->mutex, NULL); if (code) {
if (NULL == deleter->pDataBlocks) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
taosThreadMutexInit(&deleter->mutex, NULL);
*pHandle = deleter; *pHandle = deleter;
return code; return code;

20
source/libs/executor/src/dataDispatcher.c
View File

@ -104,7 +104,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
} }
int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols); int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols);
int32_t len = tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); int32_t len =
tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0);
if (len < dataLen) { if (len < dataLen) {
pEntry->compressed = 1; pEntry->compressed = 1;
pEntry->dataLen = len; pEntry->dataLen = len;
@ -168,9 +169,11 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) {
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
int32_t code = 0; int32_t code = 0;
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0); SDataDispatchBuf* pBuf;
if (NULL == pBuf) {
return TSDB_CODE_OUT_OF_MEMORY; code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf);
if (code) {
return code;
} }
if (!allocBuf(pDispatcher, pInput, pBuf)) { if (!allocBuf(pDispatcher, pInput, pBuf)) {
@ -228,7 +231,6 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRow
((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
} }
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (NULL == pDispatcher->nextOutput.pData) { if (NULL == pDispatcher->nextOutput.pData) {
@ -291,6 +293,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) {
} }
int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) {
int32_t code;
SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle));
if (NULL == dispatcher) { if (NULL == dispatcher) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -309,7 +313,11 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->pSchema = pDataSink->pInputDataBlockDesc;
dispatcher->status = DS_BUF_EMPTY; dispatcher->status = DS_BUF_EMPTY;
dispatcher->queryEnd = false; dispatcher->queryEnd = false;
dispatcher->pDataBlocks = taosOpenQueue(); code = taosOpenQueue(&dispatcher->pDataBlocks);
if (code) {
terrno = code;
goto _return;
}
taosThreadMutexInit(&dispatcher->mutex, NULL); taosThreadMutexInit(&dispatcher->mutex, NULL);
if (NULL == dispatcher->pDataBlocks) { if (NULL == dispatcher->pDataBlocks) {

18
source/libs/executor/src/groupoperator.c
View File

@ -17,6 +17,7 @@
#include "function.h" #include "function.h"
#include "os.h" #include "os.h"
#include "tname.h" #include "tname.h"
#include "tutil.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tmsg.h" #include "tmsg.h"
@ -1357,11 +1358,13 @@ static void destroyStreamPartitionOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) { int32_t initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr, SExprSupp* pTbnameExpr) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI; SStorageAPI* pAPI = &downstream->pTaskInfo->storageAPI;
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
return; return code;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
@ -1369,8 +1372,9 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup
pScanInfo->pPartScalarSup = pExpr; pScanInfo->pPartScalarSup = pExpr;
pScanInfo->pPartTbnameSup = pTbnameExpr; pScanInfo->pPartTbnameSup = pTbnameExpr;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); code = pAPI->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, 0, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
} }
return code;
} }
SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) {
@ -1421,6 +1425,7 @@ void freePartItem(void* ptr) {
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo) { SExecTaskInfo* pTaskInfo) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -1511,14 +1516,19 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState); setOperatorStreamStateFn(pOperator, streamOpReleaseState, streamOpReloadState);
initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup); code = initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup, &pInfo->tbnameCalSup);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
return pOperator; return pOperator;
_error: _error:
pTaskInfo->code = code; pTaskInfo->code = code;
destroyStreamPartitionOperatorInfo(pInfo); destroyStreamPartitionOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

88
source/libs/executor/src/scanoperator.c
View File

@ -388,9 +388,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost->totalCheckedRows += pBlock->info.rows; pCost->totalCheckedRows += pBlock->info.rows;
pCost->loadBlocks += 1; pCost->loadBlocks += 1;
SSDataBlock* p = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, NULL); SSDataBlock* p = NULL;
if (p == NULL) { int32_t code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL);
return terrno; if (p == NULL || code != TSDB_CODE_SUCCESS) {
return code;
} }
ASSERT(p == pBlock); ASSERT(p == pBlock);
@ -1351,7 +1352,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
} }
if (hasNext) { if (hasNext) {
/*SSDataBlock* p = */ pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, NULL); SSDataBlock* p = NULL;
code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL);
if (code != TSDB_CODE_SUCCESS) {
return NULL;
}
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid); pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
} }
@ -2477,27 +2483,54 @@ static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBl
} }
} }
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff) { int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) {
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); int32_t code = TSDB_CODE_SUCCESS;
len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup); int32_t lino = 0;
*pBuff = taosMemoryCalloc(1, len); int32_t len = 0;
void* buf = *pBuff; code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup); TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo);
return len; len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
*pBuff = taosMemoryCalloc(1, len);
if (!(*pBuff)) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
void* buf = *pBuff;
encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(buf, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
*pLen = len;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (!pInfo->pState) { if (!pInfo->pState) {
return; return;
} }
if (needSaveStreamOperatorInfo(&pInfo->basic)) { if (needSaveStreamOperatorInfo(&pInfo->basic)) {
void* pBuf = NULL; void* pBuf = NULL;
int32_t len = streamScanOperatorEncode(pInfo, &pBuf); int32_t len = 0;
code = streamScanOperatorEncode(pInfo, &pBuf, &len);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
saveStreamOperatorStateComplete(&pInfo->basic); saveStreamOperatorStateComplete(&pInfo->basic);
} }
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
} }
// other properties are recovered from the execution plan // other properties are recovered from the execution plan
@ -2929,8 +2962,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
} }
SSDataBlock* pBlock = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, NULL); SSDataBlock* pBlock = NULL;
if (pBlock == NULL) { code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL);
if (pBlock == NULL || code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
@ -3094,17 +3128,35 @@ static void destroyStreamScanOperatorInfo(void* param) {
} }
void streamScanReleaseState(SOperatorInfo* pOperator) { void streamScanReleaseState(SOperatorInfo* pOperator) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamScanInfo* pInfo = pOperator->info; SStreamScanInfo* pInfo = pOperator->info;
void* pBuff = NULL;
if (!pInfo->pState) { if (!pInfo->pState) {
return; return;
} }
if (!pInfo->pUpdateInfo) { if (!pInfo->pUpdateInfo) {
return; return;
} }
int32_t len = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo); int32_t len = 0;
void* pBuff = taosMemoryCalloc(1, len); code = pInfo->stateStore.updateInfoSerialize(NULL, 0, pInfo->pUpdateInfo, &len);
pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo); TSDB_CHECK_CODE(code, lino, _end);
pBuff = taosMemoryCalloc(1, len);
if (!pBuff) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _end);
}
int32_t tmp = 0;
code = pInfo->stateStore.updateInfoSerialize(pBuff, len, pInfo->pUpdateInfo, &tmp);
TSDB_CHECK_CODE(code, lino, _end);
pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME), pBuff, len);
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
taosMemoryFree(pBuff); taosMemoryFree(pBuff);
} }

90
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -68,9 +68,7 @@ void destroyStreamCountAggOperatorInfo(void* param) {
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
} }
bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { bool isSlidingCountWindow(SStreamAggSupporter* pAggSup) { return pAggSup->windowCount != pAggSup->windowSliding; }
return pAggSup->windowCount != pAggSup->windowSliding;
}
void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin, void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, SCountWindowInfo* pCurWin,
SBuffInfo* pBuffInfo) { SBuffInfo* pBuffInfo) {
@ -89,13 +87,14 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
ASSERT(pBuffInfo->pCur); ASSERT(pBuffInfo->pCur);
pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur); pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pBuffInfo->pCur);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->stateStore.streamStateCountWinAdd(pAggSup->pState, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
} }
} else { } else {
pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount); pBuffInfo->pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->windowCount);
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin, code = pAggSup->stateStore.streamStateSessionGetKVByCur(pBuffInfo->pCur, &pCurWin->winInfo.sessionWin,
(void**)&pCurWin->winInfo.pStatePos, &size); (void**)&pCurWin->winInfo.pStatePos, &size);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
@ -107,15 +106,16 @@ void setCountOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId,
pBuffInfo->rebuildWindow = true; pBuffInfo->rebuildWindow = true;
} }
} else { } else {
code = pAggSup->stateStore.streamStateCountWinAddIfNotExist( code = pAggSup->stateStore.streamStateCountWinAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin,
pAggSup->pState, &pCurWin->winInfo.sessionWin, pAggSup->windowCount, (void**)&pCurWin->winInfo.pStatePos, &size); pAggSup->windowCount,
(void**)&pCurWin->winInfo.pStatePos, &size);
} }
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
pCurWin->winInfo.isOutput = true; pCurWin->winInfo.isOutput = true;
} }
pCurWin->pWindowCount= pCurWin->pWindowCount =
(COUNT_TYPE*) ((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE))); (COUNT_TYPE*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - sizeof(COUNT_TYPE)));
if (*pCurWin->pWindowCount == pAggSup->windowCount) { if (*pCurWin->pWindowCount == pAggSup->windowCount) {
pBuffInfo->rebuildWindow = true; pBuffInfo->rebuildWindow = true;
@ -133,7 +133,7 @@ static int32_t updateCountWindowInfo(SStreamAggSupporter* pAggSup, SCountWindowI
int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated, int32_t start, int32_t rows, int32_t maxRows, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pStDeleted, bool* pRebuild) {
SSessionKey sWinKey = pWinInfo->winInfo.sessionWin; SSessionKey sWinKey = pWinInfo->winInfo.sessionWin;
int32_t num = 0; int32_t num = 0;
for (int32_t i = start; i < rows; i++) { for (int32_t i = start; i < rows; i++) {
if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) { if (pTs[i] < pWinInfo->winInfo.sessionWin.win.ekey) {
num++; num++;
@ -181,7 +181,7 @@ void getCountWinRange(SStreamAggSupporter* pAggSup, const SSessionKey* pKey, ESt
pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey); pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pAggSup->pState, pKey);
} }
SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN}; SSessionKey tmpKey = {.groupId = pKey->groupId, .win.ekey = INT64_MIN, .win.skey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, NULL, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
return; return;
@ -239,7 +239,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; TSKEY* startTsCols = (int64_t*)pStartTsCol->pData;
blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2); blockDataEnsureCapacity(pAggSup->pScanBlock, rows * 2);
SStreamStateCur* pCur = NULL; SStreamStateCur* pCur = NULL;
COUNT_TYPE slidingRows = 0; COUNT_TYPE slidingRows = 0;
for (int32_t i = 0; i < rows;) { for (int32_t i = 0; i < rows;) {
if (pInfo->ignoreExpiredData && if (pInfo->ignoreExpiredData &&
@ -269,7 +269,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
} }
getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range); getCountWinRange(pAggSup, &curWin.winInfo.sessionWin, STREAM_DELETE_DATA, &range);
range.win.skey = TMIN(startTsCols[i], range.win.skey); range.win.skey = TMIN(startTsCols[i], range.win.skey);
range.win.ekey = TMAX(startTsCols[rows-1], range.win.ekey); range.win.ekey = TMAX(startTsCols[rows - 1], range.win.ekey);
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL); appendDataToSpecialBlock(pAggSup->pScanBlock, &range.win.skey, &range.win.ekey, &uid, &range.groupId, NULL);
break; break;
@ -289,8 +289,7 @@ static void doStreamCountAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {
code = saveResult(curWin.winInfo, pStUpdated); code = saveResult(curWin.winInfo, pStUpdated);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), qError("%s do stream count aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
tstrerror(code));
break; break;
} }
} }
@ -397,13 +396,14 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
int32_t mapSize = 0; int32_t mapSize = 0;
buf = taosDecodeFixedI32(buf, &mapSize); buf = taosDecodeFixedI32(buf, &mapSize);
for (int32_t i = 0; i < mapSize; i++) { for (int32_t i = 0; i < mapSize; i++) {
SSessionKey key = {0}; SSessionKey key = {0};
SCountWindowInfo curWin = {0}; SCountWindowInfo curWin = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL}; SBuffInfo buffInfo = {.rebuildWindow = false, .winBuffOp = NONE_WINDOW, .pCur = NULL};
setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo); setCountOutputBuf(&pInfo->streamAggSup, key.win.skey, key.groupId, &curWin, &buffInfo);
buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &curWin.winInfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo,
sizeof(SResultWindowInfo));
} }
// 2.twAggSup // 2.twAggSup
@ -441,9 +441,9 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; uint64_t* gpDatas = (uint64_t*)pGroupCol->pData;
SRowBuffPos* pPos = NULL; SRowBuffPos* pPos = NULL;
int32_t size = 0; int32_t size = 0;
for (int32_t i = 0; i < pBlock->info.rows; i++) { for (int32_t i = 0; i < pBlock->info.rows; i++) {
SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]}; SSessionKey key = {.groupId = gpDatas[i], .win.skey = startDatas[i], .win.ekey = endDatas[i]};
SStreamStateCur* pCur = NULL; SStreamStateCur* pCur = NULL;
if (isSlidingCountWindow(pAggSup)) { if (isSlidingCountWindow(pAggSup)) {
pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount); pCur = pAggSup->stateStore.streamStateCountSeekKeyPrev(pAggSup->pState, &key, pAggSup->windowCount);
@ -452,7 +452,7 @@ void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
} }
while (1) { while (1) {
SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN}; SSessionKey tmpKey = {.groupId = gpDatas[i], .win.skey = INT64_MIN, .win.ekey = INT64_MIN};
int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void **)&pPos, &size); int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &tmpKey, (void**)&pPos, &size);
if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) { if (code != TSDB_CODE_SUCCESS || tmpKey.win.skey > endDatas[i]) {
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
break; break;
@ -482,7 +482,7 @@ void doDeleteCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SAr
SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]}; SSessionKey key = {.win.skey = startDatas[i], .win.ekey = endDatas[i], .groupId = gpDatas[i]};
while (1) { while (1) {
SSessionKey curWin = {0}; SSessionKey curWin = {0};
int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin); int32_t code = pAggSup->stateStore.streamStateCountGetKeyByRange(pAggSup->pState, &key, &curWin);
if (code == TSDB_CODE_FAILED) { if (code == TSDB_CODE_FAILED) {
break; break;
} }
@ -511,11 +511,11 @@ void deleteCountWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHa
} }
static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) { static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SStreamCountAggOperatorInfo* pInfo = pOperator->info; SStreamCountAggOperatorInfo* pInfo = pOperator->info;
SOptrBasicInfo* pBInfo = &pInfo->binfo; SOptrBasicInfo* pBInfo = &pInfo->binfo;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); qDebug("stask:%s %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);
if (pOperator->status == OP_EXEC_DONE) { if (pOperator->status == OP_EXEC_DONE) {
return NULL; return NULL;
@ -635,7 +635,7 @@ void streamCountReloadState(SOperatorInfo* pOperator) {
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_COUNT_OP_STATE_NAME,
strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_COUNT_OP_STATE_NAME), &pBuf, &size);
TSKEY ts = *(TSKEY*)pBuf; TSKEY ts = *(TSKEY*)pBuf;
pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
@ -650,11 +650,13 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) {
SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode; SCountWinodwPhysiNode* pCountNode = (SCountWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo)); SStreamCountAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamCountAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
pOperator->pTaskInfo = pTaskInfo; pOperator->pTaskInfo = pTaskInfo;
@ -664,18 +666,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
} }
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->twAggSup = (STimeWindowAggSupp){ pInfo->twAggSup = (STimeWindowAggSupp){
.waterMark = pCountNode->window.watermark, .waterMark = pCountNode->window.watermark,
@ -686,12 +684,11 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
}; };
pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId;
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState,
pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
&pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowCount = pCountNode->windowCount;
pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding;
@ -709,7 +706,8 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0; pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) { if (!pInfo->historyWins) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
@ -735,8 +733,12 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState); setOperatorStreamStateFn(pOperator, streamCountReleaseState, streamCountReloadState);
if (downstream) { if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
} }
return pOperator; return pOperator;
@ -747,6 +749,6 @@ _error:
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

112
source/libs/executor/src/streameventwindowoperator.c
View File

@ -28,9 +28,9 @@
#include "tlog.h" #include "tlog.h"
#include "ttime.h" #include "ttime.h"
#define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT) #define IS_NORMAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT)
#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" #define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState"
#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" #define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint"
typedef struct SEventWinfowFlag { typedef struct SEventWinfowFlag {
bool startFlag; bool startFlag;
@ -86,10 +86,11 @@ void destroyStreamEventOperatorInfo(void* param) {
void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) {
char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize);
pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; pWinInfo->pWinFlag = (SEventWinfowFlag*)pFlagInfo;
} }
void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos,
SEventWindowInfo* pWinInfo) {
pWinInfo->winInfo.sessionWin = *pKey; pWinInfo->winInfo.sessionWin = *pKey;
pWinInfo->winInfo.pStatePos = pPos; pWinInfo->winInfo.pStatePos = pPos;
setEventWindowFlag(pAggSup, pWinInfo); setEventWindowFlag(pAggSup, pWinInfo);
@ -122,22 +123,23 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
pCurWin->winInfo.sessionWin.groupId = groupId; pCurWin->winInfo.sessionWin.groupId = groupId;
pCurWin->winInfo.sessionWin.win.skey = ts; pCurWin->winInfo.sessionWin.win.skey = ts;
pCurWin->winInfo.sessionWin.win.ekey = ts; pCurWin->winInfo.sessionWin.win.ekey = ts;
SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); SStreamStateCur* pCur =
SSessionKey leftWinKey = {.groupId = groupId}; pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin);
void* pVal = NULL; SSessionKey leftWinKey = {.groupId = groupId};
int32_t len = 0; void* pVal = NULL;
int32_t len = 0;
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win)) {
bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0);
setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin);
if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { if (inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag)) {
pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin); pCurWin->winInfo.isOutput = !isWindowIncomplete(pCurWin);
goto _end; goto _end;
} }
} }
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin);
SSessionKey rightWinKey = {.groupId = groupId}; SSessionKey rightWinKey = {.groupId = groupId};
code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len);
bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0);
if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) {
@ -149,7 +151,7 @@ void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupI
} }
} }
SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId};
pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len);
setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin);
pCurWin->pWinFlag->startFlag = start; pCurWin->pWinFlag->startFlag = start;
@ -164,7 +166,7 @@ _end:
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
SET_SESSION_WIN_KEY_INVALID(pNextWinKey); SET_SESSION_WIN_KEY_INVALID(pNextWinKey);
} }
if(pCurWin->winInfo.pStatePos->needFree) { if (pCurWin->winInfo.pStatePos->needFree) {
pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);
} }
pAggSup->stateStore.streamStateFreeCur(pCur); pAggSup->stateStore.streamStateFreeCur(pCur);
@ -173,14 +175,14 @@ _end:
} }
int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey,
TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start,
SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { SSHashObj* pResultRows, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) {
*pRebuild = false; *pRebuild = false;
if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { if (!pWinInfo->pWinFlag->startFlag && !(starts[start])) {
return 1; return 1;
} }
TSKEY maxTs = INT64_MAX; TSKEY maxTs = INT64_MAX;
STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win;
if (pWinInfo->pWinFlag->endFlag) { if (pWinInfo->pWinFlag->endFlag) {
maxTs = pWin->ekey + 1; maxTs = pWin->ekey + 1;
@ -219,7 +221,7 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
if (ends[i]) { if (ends[i]) {
if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey) {
*pRebuild = true; *pRebuild = true;
} }
return i + 1 - start; return i + 1 - start;
@ -230,26 +232,28 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated,
SSHashObj* pStDeleted, bool addGap) { SSHashObj* pStDeleted, bool addGap) {
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;
int32_t winNum = 0; int32_t winNum = 0;
SStreamEventAggOperatorInfo* pInfo = pOperator->info; SStreamEventAggOperatorInfo* pInfo = pOperator->info;
SResultRow* pCurResult = NULL; SResultRow* pCurResult = NULL;
int32_t numOfOutput = pOperator->exprSupp.numOfExprs; int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
while (1) { while (1) {
if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) {
break; break;
} }
SEventWindowInfo nextWinInfo = {0}; SEventWindowInfo nextWinInfo = {0};
getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo);
if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) ||
!inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) {
releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);
break; break;
} }
setEventWindowFlag(pAggSup, &nextWinInfo); setEventWindowFlag(pAggSup, &nextWinInfo);
compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated,
pStDeleted, false);
pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag;
winNum++; winNum++;
} }
@ -275,7 +279,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winRows = 0; int32_t winRows = 0;
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pColStart = NULL; SColumnInfoData* pColStart = NULL;
SColumnInfoData* pColEnd = NULL; SColumnInfoData* pColEnd = NULL;
pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);
pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
@ -290,7 +294,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
return; return;
} }
SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart); code = filterSetDataFromSlotId(pInfo->pStartCondInfo, &paramStart);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("set data from start slotId error."); qError("set data from start slotId error.");
@ -299,7 +304,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t statusStart = 0; int32_t statusStart = 0;
filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart);
SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock),
.pDataBlock = pSDataBlock->pDataBlock};
code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd); code = filterSetDataFromSlotId(pInfo->pEndCondInfo, &paramEnd);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("set data from end slotId error."); qError("set data from end slotId error.");
@ -320,20 +326,23 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
int32_t winIndex = 0; int32_t winIndex = 0;
bool allEqual = true; bool allEqual = true;
SEventWindowInfo curWin = {0}; SEventWindowInfo curWin = {0};
SSessionKey nextWinKey = {0}; SSessionKey nextWinKey = {0};
setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin,
&nextWinKey);
setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);
bool rebuild = false; bool rebuild = false;
winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, winRows =
pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData,
rows, i, pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild);
ASSERT(winRows >= 1); ASSERT(winRows >= 1);
if (rebuild) { if (rebuild) {
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) && !isWindowIncomplete(&curWin)) { if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_EVENT_OP(pOperator) &&
!isWindowIncomplete(&curWin)) {
saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin); saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);
} }
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -603,7 +612,8 @@ void streamEventReleaseState(SOperatorInfo* pOperator) {
char* pBuff = taosMemoryCalloc(1, resSize); char* pBuff = taosMemoryCalloc(1, resSize);
memcpy(pBuff, pInfo->historyWins->pData, winSize); memcpy(pBuff, pInfo->historyWins->pData, winSize);
memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));
qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); qDebug("===stream=== event window operator relase state. save result count:%d",
(int32_t)taosArrayGetSize(pInfo->historyWins));
pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME,
strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize);
pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);
@ -647,7 +657,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
pSeKeyBuf[i].groupId, i); pSeKeyBuf[i].groupId, i);
getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo);
//event window has been deleted // event window has been deleted
if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (!IS_VALID_SESSION_WIN(curInfo.winInfo)) {
continue; continue;
} }
@ -659,7 +669,7 @@ void streamEventReloadState(SOperatorInfo* pOperator) {
compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false);
qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,
curInfo.winInfo.sessionWin.groupId); curInfo.winInfo.sessionWin.groupId);
if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {
saveSessionOutputBuf(pAggSup, &curInfo.winInfo); saveSessionOutputBuf(pAggSup, &curInfo.winInfo);
} }
@ -693,6 +703,7 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode;
int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -783,20 +794,18 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
return pOperator; return pOperator;
@ -804,5 +813,6 @@ _error:
destroyStreamEventOperatorInfo(pInfo); destroyStreamEventOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }

192
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -28,14 +28,18 @@
#include "tlog.h" #include "tlog.h"
#include "ttime.h" #include "ttime.h"
#define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) #define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) #define IS_MID_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL)
#define IS_NORMAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) #define IS_NORMAL_INTERVAL_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL)
#define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) #define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || (op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) #define IS_NORMAL_SESSION_OP(op) \
((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || \
(op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION)
#define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) #define IS_NORMAL_STATE_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE)
#define DEAULT_DELETE_MARK INT64_MAX #define DEAULT_DELETE_MARK INT64_MAX
#define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState"
@ -263,14 +267,16 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa
if (chIds) { if (chIds) {
int32_t childId = getChildIndex(pBlock); int32_t childId = getChildIndex(pBlock);
if (pInvalidWins) { if (pInvalidWins) {
qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts, winRes.groupId, childId); qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,
winRes.groupId, childId);
taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0); taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);
} }
SArray* chArray = *(void**)chIds; SArray* chArray = *(void**)chIds;
int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ); int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);
if (index != -1) { if (index != -1) {
qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey, winGpId, childId); qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,
winGpId, childId);
getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC); getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);
continue; continue;
} }
@ -390,7 +396,7 @@ static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWin
} }
void destroyFlusedPos(void* pRes) { void destroyFlusedPos(void* pRes) {
SRowBuffPos* pPos = (SRowBuffPos*) pRes; SRowBuffPos* pPos = (SRowBuffPos*)pRes;
if (!pPos->needFree && !pPos->pRowBuff) { if (!pPos->needFree && !pPos->pRowBuff) {
taosMemoryFreeClear(pPos->pKey); taosMemoryFreeClear(pPos->pKey);
taosMemoryFree(pPos); taosMemoryFree(pPos);
@ -488,8 +494,11 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
pScanInfo->windowSup.parentType = type; pScanInfo->windowSup.parentType = type;
pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup; pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = int32_t code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,
pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at %d since %s", __func__, __LINE__, tstrerror(code));
}
} }
pScanInfo->interval = pInfo->interval; pScanInfo->interval = pInfo->interval;
@ -624,17 +633,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
} }
static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval, SArray* pPullWins, static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,
int32_t numOfCh, SOperatorInfo* pOperator) { SArray* pPullWins, int32_t numOfCh, SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
TSKEY* tsData = (TSKEY*)pStartCol->pData; TSKEY* tsData = (TSKEY*)pStartCol->pData;
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
TSKEY* tsEndData = (TSKEY*)pEndCol->pData; TSKEY* tsEndData = (TSKEY*)pEndCol->pData;
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; uint64_t* groupIdData = (uint64_t*)pGroupCol->pData;
int32_t chId = getChildIndex(pBlock); int32_t chId = getChildIndex(pBlock);
bool res = false; bool res = false;
for (int32_t i = 0; i < pBlock->info.rows; i++) { for (int32_t i = 0; i < pBlock->info.rows; i++) {
TSKEY winTs = tsData[i]; TSKEY winTs = tsData[i];
while (winTs <= tsEndData[i]) { while (winTs <= tsEndData[i]) {
@ -650,7 +659,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
// pull data is over // pull data is over
taosArrayDestroy(chArray); taosArrayDestroy(chArray);
taosHashRemove(pMap, &winRes, sizeof(SWinKey)); taosHashRemove(pMap, &winRes, sizeof(SWinKey));
res =true; res = true;
qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts); qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);
void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey)); void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));
@ -663,7 +672,8 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
.calWin.skey = nextWin.skey, .calWin.skey = nextWin.skey,
.calWin.ekey = nextWin.skey}; .calWin.ekey = nextWin.skey};
// add pull data request // add pull data request
qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d", winRes.ts, winRes.groupId, numOfCh); qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",
winRes.ts, winRes.groupId, numOfCh);
if (IS_MID_INTERVAL_OP(pOperator)) { if (IS_MID_INTERVAL_OP(pOperator)) {
SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info; SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;
taosArrayPush(pInfo->pMidPullDatas, &winRes); taosArrayPush(pInfo->pMidPullDatas, &winRes);
@ -671,7 +681,7 @@ static bool processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFina
taosArrayPush(pInfo->pDelWins, &winRes); taosArrayPush(pInfo->pDelWins, &winRes);
addPullWindow(pMap, &winRes, numOfCh); addPullWindow(pMap, &winRes, numOfCh);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
} }
qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh); qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);
} }
@ -690,7 +700,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i); SWinKey* winKey = taosArrayGet(wins, i);
STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) { if (!chIds) {
SPullWindowInfo pull = { SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
@ -698,7 +708,7 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, i
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild); addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,winKey, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);
} }
qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild); qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);
} }
@ -751,7 +761,8 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDat
if (pBlock->info.id.groupId == 0) { if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = groupId; pBlock->info.id.groupId = groupId;
void* tbname = NULL; void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname, false) < 0) { if (pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
false) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -829,9 +840,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN
return startPos; return startPos;
} }
bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }
return pInfo->primaryPkIndex != -1;
}
static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId, static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,
SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) { SSHashObj* pUpdatedMap, SSHashObj* pDeletedMap) {
@ -852,23 +861,22 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData; tsCols = (int64_t*)pColDataInfo->pData;
void* pPkVal = NULL; void* pPkVal = NULL;
int32_t pkLen = 0; int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL; SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) { if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
} }
if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {
qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,
",maxKey %" PRId64, pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);
// timestamp of the data is incorrect // timestamp of the data is incorrect
if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {
qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,
pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey);
} }
} }
@ -916,7 +924,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild); addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);
if (pInfo->destHasPrimaryKey) { if (pInfo->destHasPrimaryKey) {
tSimpleHashPut(pInfo->pDeletedMap,&winRes, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);
} }
} }
} else { } else {
@ -962,7 +970,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
}; };
if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) { if (pInfo->destHasPrimaryKey && code == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {
tSimpleHashPut(pDeletedMap,&key, sizeof(SWinKey), NULL, 0); tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);
} }
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
@ -1365,7 +1373,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
pInfo->binfo.pRes->info.type = pBlock->info.type; pInfo->binfo.pRes->info.type = pBlock->info.type;
} else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
pBlock->info.type == STREAM_CLEAR) { pBlock->info.type == STREAM_CLEAR) {
SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); SArray* delWins = taosArrayInit(8, sizeof(SWinKey));
SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL; SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;
doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap); doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);
if (IS_FINAL_INTERVAL_OP(pOperator)) { if (IS_FINAL_INTERVAL_OP(pOperator)) {
@ -1399,7 +1407,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);
continue; continue;
} else if (pBlock->info.type == STREAM_RETRIEVE) { } else if (pBlock->info.type == STREAM_RETRIEVE) {
if(!IS_FINAL_INTERVAL_OP(pOperator)) { if (!IS_FINAL_INTERVAL_OP(pOperator)) {
pInfo->recvRetrive = true; pInfo->recvRetrive = true;
copyDataBlock(pInfo->pMidRetriveRes, pBlock); copyDataBlock(pInfo->pMidRetriveRes, pBlock);
pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE; pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;
@ -1525,8 +1533,8 @@ void streamIntervalReloadState(SOperatorInfo* pOperator) {
SStreamIntervalOperatorInfo* pInfo = pOperator->info; SStreamIntervalOperatorInfo* pInfo = pOperator->info;
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL && if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&
pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) { pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME, int32_t code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,
strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_INTERVAL_OP_STATE_NAME), &pBuf, &size);
if (code == 0) { if (code == 0) {
@ -1600,7 +1608,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex); pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pInfo->pState, &pTaskInfo->storageAPI.functionStore); pInfo->pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
} }
@ -1755,27 +1763,37 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num
} }
} }
void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,
STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) { STimeWindowAggSupp* pTwSup, struct SSteamOpBasicInfo* pBasic) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {
SStreamPartitionOperatorInfo* pScanInfo = downstream->info; SStreamPartitionOperatorInfo* pScanInfo = downstream->info;
pScanInfo->tsColIndex = tsColIndex; pScanInfo->tsColIndex = tsColIndex;
} }
if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic); code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);
return; return code;
} }
SStreamScanInfo* pScanInfo = downstream->info; SStreamScanInfo* pScanInfo = downstream->info;
pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type}; pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};
pScanInfo->pState = pAggSup->pState; pScanInfo->pState = pAggSup->pState;
if (!pScanInfo->pUpdateInfo) { if (!pScanInfo->pUpdateInfo) {
pScanInfo->pUpdateInfo = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark, code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,
pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen); pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,
&pScanInfo->pUpdateInfo);
TSDB_CHECK_CODE(code, lino, _end);
} }
pScanInfo->twAggSup = *pTwSup; pScanInfo->twAggSup = *pTwSup;
pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo; pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;
pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex; pBasic->primaryPkIndex = pScanInfo->primaryKeyIndex;
_end:
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
}
return code;
} }
static TSKEY sesionTs(void* pKey) { static TSKEY sesionTs(void* pKey) {
@ -2148,8 +2166,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
TSKEY* endTsCols = (int64_t*)pEndTsCol->pData; TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
void* pPkVal = NULL; void* pPkVal = NULL;
int32_t pkLen = 0; int32_t pkLen = 0;
SColumnInfoData* pPkColDataInfo = NULL; SColumnInfoData* pPkColDataInfo = NULL;
if (hasSrcPrimaryKeyCol(&pInfo->basic)) { if (hasSrcPrimaryKeyCol(&pInfo->basic)) {
pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);
@ -2454,8 +2472,8 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa
pBlock->info.id.groupId = pKey->groupId; pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL; void* tbname = NULL;
if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, if (pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,
&tbname, false) < 0) { false) < 0) {
pBlock->info.parTbName[0] = 0; pBlock->info.parTbName[0] = 0;
} else { } else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -2935,7 +2953,7 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -2968,7 +2986,7 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,
strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_SESSION_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY));
@ -3018,6 +3036,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;
int32_t numOfCols = 0; int32_t numOfCols = 0;
int32_t code = TSDB_CODE_OUT_OF_MEMORY; int32_t code = TSDB_CODE_OUT_OF_MEMORY;
int32_t lino = 0;
SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -3112,8 +3131,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState); setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);
if (downstream) { if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
TSDB_CHECK_CODE(code, lino, _error);
} }
return pOperator; return pOperator;
@ -3124,6 +3147,7 @@ _error:
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }
@ -3574,7 +3598,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
if (!allEqual) { if (!allEqual) {
uint64_t uid = 0; uint64_t uid = 0;
appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,
&curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL);
tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin); doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);
releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
@ -3681,9 +3705,9 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
SSessionKey key = {0}; SSessionKey key = {0};
SResultWindowInfo winfo = {0}; SResultWindowInfo winfo = {0};
buf = decodeSSessionKey(buf, &key); buf = decodeSSessionKey(buf, &key);
pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &winfo.sessionWin, NULL, pAggSup->stateKeySize,
pAggSup->stateKeySize, compareStateKey, compareStateKey, (void**)&winfo.pStatePos,
(void**)&winfo.pStatePos, &pAggSup->resultRowSize); &pAggSup->resultRowSize);
buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);
tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo));
} }
@ -3877,7 +3901,7 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
int32_t size = 0; int32_t size = 0;
void* pBuf = NULL; void* pBuf = NULL;
int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME, int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,
strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size); strlen(STREAM_STATE_OP_STATE_NAME), &pBuf, &size);
int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);
qDebug("===stream=== reload state. get result count:%d", num); qDebug("===stream=== reload state. get result count:%d", num);
SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;
@ -3941,12 +3965,13 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey); SColumnNode* pColNode = (SColumnNode*)(pStateNode->pStateKey);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo)); SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _error; TSDB_CHECK_CODE(code, lino, _error);
} }
pInfo->stateCol = extractColumnFromColumnNode(pColNode); pInfo->stateCol = extractColumnFromColumnNode(pColNode);
@ -3955,9 +3980,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
} }
pInfo->twAggSup = (STimeWindowAggSupp){ pInfo->twAggSup = (STimeWindowAggSupp){
@ -3984,9 +4007,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize,
type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, type, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup,
GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex); GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI, pInfo->primaryTsIndex);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL; pInfo->pDelIterator = NULL;
@ -3999,7 +4021,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->dataVersion = 0; pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) { if (!pInfo->historyWins) {
goto _error; code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _error);
} }
if (pHandle) { if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory; pInfo->isHistoryOp = pHandle->fillHistory;
@ -4026,17 +4049,20 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState); setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup, &pInfo->basic); code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,
&pInfo->twAggSup, &pInfo->basic);
TSDB_CHECK_CODE(code, lino, _error);
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
return pOperator; return pOperator;
_error: _error:
destroyStreamStateOperatorInfo(pInfo); destroyStreamStateOperatorInfo(pInfo);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
pTaskInfo->code = code; pTaskInfo->code = code;
qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
return NULL; return NULL;
} }
@ -4203,11 +4229,12 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
.precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,
}; };
pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark, pInfo->twAggSup =
.calTrigger = pIntervalPhyNode->window.triggerType, (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,
.maxTs = INT64_MIN, .calTrigger = pIntervalPhyNode->window.triggerType,
.minTs = INT64_MAX, .maxTs = INT64_MIN,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)}; .minTs = INT64_MAX,
.deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};
ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay"); ASSERTS(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY, "trigger type should not be max delay");
@ -4335,7 +4362,7 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
.ts = nextWin.skey, .ts = nextWin.skey,
.groupId = groupId, .groupId = groupId,
}; };
void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey)); void* chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));
int32_t index = -1; int32_t index = -1;
SArray* chArray = NULL; SArray* chArray = NULL;
int32_t chId = 0; int32_t chId = 0;
@ -4407,8 +4434,8 @@ static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pS
static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) { static void addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {
int32_t size = taosArrayGetSize(wins); int32_t size = taosArrayGetSize(wins);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
SWinKey* winKey = taosArrayGet(wins, i); SWinKey* winKey = taosArrayGet(wins, i);
void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey)); void* chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));
if (!chIds) { if (!chIds) {
addPullWindow(pMidPullMap, winKey, numOfChild); addPullWindow(pMidPullMap, winKey, numOfChild);
qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild); qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);
@ -4611,5 +4638,6 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
void setStreamOperatorCompleted(SOperatorInfo* pOperator) { void setStreamOperatorCompleted(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator); setOperatorCompleted(pOperator);
qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),
getStreamOpName(pOperator->operatorType), pOperator->status);
} }

1
source/libs/executor/src/timewindowoperator.c
View File

@ -1139,7 +1139,6 @@ void destroyIntervalOperatorInfo(void* param) {
taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroy(pInfo->pInterpCols);
pInfo->pInterpCols = NULL; pInfo->pInterpCols = NULL;
taosArrayDestroyEx(pInfo->pPrevValues, freeItem); taosArrayDestroyEx(pInfo->pPrevValues, freeItem);
pInfo->pPrevValues = NULL; pInfo->pPrevValues = NULL;
cleanupGroupResInfo(&pInfo->groupResInfo); cleanupGroupResInfo(&pInfo->groupResInfo);

4
source/libs/parser/inc/parTranslater.h
View File

@ -47,7 +47,7 @@ typedef struct STranslateContext {
SNode* pPostRoot; SNode* pPostRoot;
} STranslateContext; } STranslateContext;
bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode); int32_t biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode, bool* pRet);
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect); int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt); int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt);
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput); int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
@ -57,4 +57,4 @@ int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, c
} }
#endif #endif
#endif /*_TD_PARSER_TRANS_H_*/ #endif /*_TD_PARSER_TRANS_H_*/

2
source/libs/parser/inc/sql.y
View File

@ -509,7 +509,7 @@ cmd ::= SHOW USERS FULL.
cmd ::= SHOW USER PRIVILEGES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } cmd ::= SHOW USER PRIVILEGES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); }
cmd ::= SHOW db_kind_opt(A) DATABASES. { cmd ::= SHOW db_kind_opt(A) DATABASES. {
pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT);
setShowKind(pCxt, pCxt->pRootNode, A); (void)setShowKind(pCxt, pCxt->pRootNode, A);
} }
cmd ::= SHOW table_kind_db_name_cond_opt(A) TABLES like_pattern_opt(B). { cmd ::= SHOW table_kind_db_name_cond_opt(A) TABLES like_pattern_opt(B). {
pCxt->pRootNode = createShowTablesStmt(pCxt, A, B, OP_TYPE_LIKE); pCxt->pRootNode = createShowTablesStmt(pCxt, A, B, OP_TYPE_LIKE);

43
source/libs/parser/src/parAstCreater.c
View File

@ -101,7 +101,7 @@ static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken,
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG); pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG);
} else { } else {
strncpy(pPassword, pPasswordToken->z, pPasswordToken->n); strncpy(pPassword, pPasswordToken->z, pPasswordToken->n);
strdequote(pPassword); (void)strdequote(pPassword);
if (strtrim(pPassword) <= 0) { if (strtrim(pPassword) <= 0) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY); pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_PASSWD_EMPTY);
} else if (invalidPassword(pPassword)) { } else if (invalidPassword(pPassword)) {
@ -126,8 +126,8 @@ static int32_t parseEndpoint(SAstCreateContext* pCxt, const SToken* pEp, char* p
char ep[TSDB_FQDN_LEN + 1 + 5] = {0}; char ep[TSDB_FQDN_LEN + 1 + 5] = {0};
COPY_STRING_FORM_ID_TOKEN(ep, pEp); COPY_STRING_FORM_ID_TOKEN(ep, pEp);
strdequote(ep); (void)strdequote(ep);
strtrim(ep); (void)strtrim(ep);
if (NULL == pPort) { if (NULL == pPort) {
strcpy(pFqdn, ep); strcpy(pFqdn, ep);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -371,7 +371,7 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
val->literal = strndup(pLiteral->z, pLiteral->n); val->literal = strndup(pLiteral->z, pLiteral->n);
if (TK_NK_ID != pLiteral->type && TK_TIMEZONE != pLiteral->type && if (TK_NK_ID != pLiteral->type && TK_TIMEZONE != pLiteral->type &&
(IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) { (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n); (void)trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
} }
CHECK_OUT_OF_MEM(val->literal); CHECK_OUT_OF_MEM(val->literal);
val->node.resType.type = dataType; val->node.resType.type = dataType;
@ -784,7 +784,10 @@ SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLitera
return NULL; return NULL;
} }
} }
taosArrayPush(pCxt->pPlaceholderValues, &val); if (NULL == taosArrayPush(pCxt->pPlaceholderValues, &val)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
}
CHECK_PARSER_STATUS(pCxt);
return (SNode*)val; return (SNode*)val;
} }
@ -905,7 +908,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d
} else if (TSDB_DATA_TYPE_NCHAR == dt.type) { } else if (TSDB_DATA_TYPE_NCHAR == dt.type) {
func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE;
} }
nodesListMakeAppend(&func->pParameterList, pExpr); pCxt->errCode = nodesListMakeAppend(&func->pParameterList, pExpr);
CHECK_PARSER_STATUS(pCxt);
return (SNode*)func; return (SNode*)func;
} }
@ -923,8 +927,10 @@ SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2) {
CHECK_OUT_OF_MEM(list); CHECK_OUT_OF_MEM(list);
list->pNodeList = nodesMakeList(); list->pNodeList = nodesMakeList();
CHECK_OUT_OF_MEM(list->pNodeList); CHECK_OUT_OF_MEM(list->pNodeList);
nodesListAppend(list->pNodeList, p1); pCxt->errCode = nodesListAppend(list->pNodeList, p1);
nodesListAppend(list->pNodeList, p2); CHECK_PARSER_STATUS(pCxt);
pCxt->errCode = nodesListAppend(list->pNodeList, p2);
CHECK_PARSER_STATUS(pCxt);
return (SNode*)list; return (SNode*)list;
} }
@ -1121,7 +1127,8 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) {
CHECK_OUT_OF_MEM(groupingSet); CHECK_OUT_OF_MEM(groupingSet);
groupingSet->groupingSetType = GP_TYPE_NORMAL; groupingSet->groupingSetType = GP_TYPE_NORMAL;
groupingSet->pParameterList = nodesMakeList(); groupingSet->pParameterList = nodesMakeList();
nodesListAppend(groupingSet->pParameterList, pNode); pCxt->errCode = nodesListAppend(groupingSet->pParameterList, pNode);
CHECK_PARSER_STATUS(pCxt);
return (SNode*)groupingSet; return (SNode*)groupingSet;
} }
@ -1842,7 +1849,7 @@ SNode* createCreateSubTableFromFileClause(SAstCreateContext* pCxt, bool ignoreEx
pStmt->ignoreExists = ignoreExists; pStmt->ignoreExists = ignoreExists;
pStmt->pSpecificTags = pSpecificTags; pStmt->pSpecificTags = pSpecificTags;
if (TK_NK_STRING == pFilePath->type) { if (TK_NK_STRING == pFilePath->type) {
trimString(pFilePath->z, pFilePath->n, pStmt->filePath, PATH_MAX); (void)trimString(pFilePath->z, pFilePath->n, pStmt->filePath, PATH_MAX);
} else { } else {
strncpy(pStmt->filePath, pFilePath->z, pFilePath->n); strncpy(pStmt->filePath, pFilePath->z, pFilePath->n);
} }
@ -2087,7 +2094,7 @@ SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, S
pDbName = createIdentifierValueNode(pCxt, &option.dbName); pDbName = createIdentifierValueNode(pCxt, &option.dbName);
} }
SNode* pStmt = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, pDbName, pTbName, tableCondType); SNode* pStmt = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, pDbName, pTbName, tableCondType);
setShowKind(pCxt, pStmt, option.kind); (void)setShowKind(pCxt, pStmt, option.kind);
return pStmt; return pStmt;
} }
@ -2408,9 +2415,9 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
} else { } else {
pStmt->dnodeId = -1; pStmt->dnodeId = -1;
} }
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) { if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
} }
return (SNode*)pStmt; return (SNode*)pStmt;
} }
@ -2591,9 +2598,9 @@ SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, co
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SAlterClusterStmt* pStmt = (SAlterClusterStmt*)nodesMakeNode(QUERY_NODE_ALTER_CLUSTER_STMT); SAlterClusterStmt* pStmt = (SAlterClusterStmt*)nodesMakeNode(QUERY_NODE_ALTER_CLUSTER_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) { if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
} }
return (SNode*)pStmt; return (SNode*)pStmt;
} }
@ -2602,9 +2609,9 @@ SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, cons
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SAlterLocalStmt* pStmt = (SAlterLocalStmt*)nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT); SAlterLocalStmt* pStmt = (SAlterLocalStmt*)nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config)); (void)trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) { if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value)); (void)trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
} }
return (SNode*)pStmt; return (SNode*)pStmt;
} }
@ -2869,7 +2876,7 @@ SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SKillQueryStmt* pStmt = (SKillQueryStmt*)nodesMakeNode(QUERY_NODE_KILL_QUERY_STMT); SKillQueryStmt* pStmt = (SKillQueryStmt*)nodesMakeNode(QUERY_NODE_KILL_QUERY_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1); (void)trimString(pQueryId->z, pQueryId->n, pStmt->queryId, sizeof(pStmt->queryId) - 1);
return (SNode*)pStmt; return (SNode*)pStmt;
} }

101
source/libs/parser/src/parInsertSml.c
View File

@ -112,7 +112,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
for (int i = 0; i < tags->numOfBound; ++i) { for (int i = 0; i < tags->numOfBound; ++i) {
SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
SSmlKv* kv = taosArrayGet(cols, i); SSmlKv* kv = taosArrayGet(cols, i);
if (kv == NULL){
code = TSDB_CODE_SML_INVALID_DATA;
uError("SML smlBuildTagRow error kv is null");
goto end;
}
if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 ||
kv->type != pTagSchema->type) { kv->type != pTagSchema->type) {
code = TSDB_CODE_SML_INVALID_DATA; code = TSDB_CODE_SML_INVALID_DATA;
@ -120,7 +124,11 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end; goto end;
} }
taosArrayPush(*tagName, pTagSchema->name); if (taosArrayPush(*tagName, pTagSchema->name) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag name");
goto end;
}
STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
// strcpy(val.colName, pTagSchema->name); // strcpy(val.colName, pTagSchema->name);
if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY || if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
@ -141,7 +149,7 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
goto end; goto end;
} }
char buf[512] = {0}; char buf[512] = {0};
snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); (void)snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
taosMemoryFree(p); taosMemoryFree(p);
code = buildSyntaxErrMsg(msg, buf, kv->value); code = buildSyntaxErrMsg(msg, buf, kv->value);
goto end; goto end;
@ -149,9 +157,13 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem
val.pData = p; val.pData = p;
val.nData = output; val.nData = output;
} else { } else {
memcpy(&val.i64, &(kv->value), kv->length); (void)memcpy(&val.i64, &(kv->value), kv->length);
}
if (taosArrayPush(pTagArray, &val) == NULL){
code = TSDB_CODE_OUT_OF_MEMORY;
uError("SML smlBuildTagRow error push tag array");
goto end;
} }
taosArrayPush(pTagArray, &val);
} }
code = tTagNew(pTagArray, 1, false, ppTag); code = tTagNew(pTagArray, 1, false, ppTag);
@ -166,20 +178,19 @@ end:
return code; return code;
} }
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {
STableDataCxt* pTableCxt = NULL;
SVCreateTbReq* pCreateTbReq = NULL; SVCreateTbReq* pCreateTbReq = NULL;
int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return NULL; return ret;
} }
ret = initTableColSubmitData(pTableCxt); ret = initTableColSubmitData(*cxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return NULL; return ret;
} }
return pTableCxt; return TSDB_CODE_SUCCESS;
} }
void clearColValArraySml(SArray* pCols) { void clearColValArraySml(SArray* pCols) {
@ -197,6 +208,9 @@ void clearColValArraySml(SArray* pCols) {
int32_t smlBuildRow(STableDataCxt* pTableCxt) { int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (pRow == NULL){
return TSDB_CODE_OUT_OF_MEMORY;
}
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) { if (TSDB_CODE_SUCCESS != ret) {
return ret; return ret;
@ -211,18 +225,22 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
int ret = TSDB_CODE_SUCCESS; int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index; SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
if (pVal == NULL) {
return TSDB_CODE_SUCCESS;
}
SSmlKv* kv = (SSmlKv*)data; SSmlKv* kv = (SSmlKv*)data;
if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
kv->type != pColSchema->type) { kv->type != pColSchema->type) {
ret = TSDB_CODE_SML_INVALID_DATA; ret = TSDB_CODE_SML_INVALID_DATA;
char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
if (tmp) { if (tmp) {
memcpy(tmp, kv->key, kv->keyLen); (void)memcpy(tmp, kv->key, kv->keyLen);
uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
pColSchema->name, tDataTypes[pColSchema->type].name); pColSchema->name, tDataTypes[pColSchema->type].name);
taosMemoryFree(tmp); taosMemoryFree(tmp);
} else { } else {
uError("SML smlBuildCol out of memory"); uError("SML smlBuildCol out of memory");
ret = TSDB_CODE_OUT_OF_MEMORY;
} }
goto end; goto end;
} }
@ -256,9 +274,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length; pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length); pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else { } else {
memcpy(&pVal->value.val, &(kv->value), kv->length); (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
} }
pVal->flag = CV_FLAG_VALUE; pVal->flag = CV_FLAG_VALUE;
@ -276,8 +294,13 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
SVCreateTbReq* pCreateTblReq = NULL; SVCreateTbReq* pCreateTblReq = NULL;
SArray* tagName = NULL; SArray* tagName = NULL;
insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); int ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "init bound cols error");
goto end;
}
ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "bound tags error"); buildInvalidOperationMsg(&pBuf, "bound tags error");
goto end; goto end;
@ -299,7 +322,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
ttl); ttl);
pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); if (pCreateTblReq->ctb.stbName == NULL){
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if (dataFormat) { if (dataFormat) {
STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
@ -345,11 +372,18 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int32_t r = 0; r < rowNum; ++r) { for (int32_t r = 0; r < rowNum; ++r) {
void* rowData = taosArrayGetP(cols, r); void* rowData = taosArrayGetP(cols, r);
if (rowData == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
// 1. set the parsed value from sql string // 1. set the parsed value from sql string
for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
if (pVal == NULL) {
ret = TSDB_CODE_SML_INVALID_DATA;
goto end;
}
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
if (p == NULL) { if (p == NULL) {
continue; continue;
@ -388,14 +422,22 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) { } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
pVal->value.nData = kv->length; pVal->value.nData = kv->length;
pVal->value.pData = taosMemoryMalloc(kv->length); pVal->value.pData = taosMemoryMalloc(kv->length);
memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length); if (NULL == pVal->value.pData) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
} else { } else {
memcpy(&pVal->value.val, &(kv->value), kv->length); (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
} }
pVal->flag = CV_FLAG_VALUE; pVal->flag = CV_FLAG_VALUE;
} }
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
if (NULL == pRow) {
ret = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) { if (TSDB_CODE_SUCCESS != ret) {
buildInvalidOperationMsg(&pBuf, "tRowBuild error"); buildInvalidOperationMsg(&pBuf, "tRowBuild error");
@ -415,11 +457,12 @@ end:
return ret; return ret;
} }
SQuery* smlInitHandle() { int32_t smlInitHandle(SQuery** query) {
*query = NULL;
SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == pQuery) { if (NULL == pQuery) {
uError("create pQuery error"); uError("create pQuery error");
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false; pQuery->haveResultSet = false;
@ -428,14 +471,22 @@ SQuery* smlInitHandle() {
if (NULL == stmt) { if (NULL == stmt) {
uError("create SVnodeModifyOpStmt error"); uError("create SVnodeModifyOpStmt error");
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
return NULL; return TSDB_CODE_OUT_OF_MEMORY;
} }
stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (stmt->pTableBlockHashObj == NULL){
uError("create pTableBlockHashObj error");
qDestroyQuery(pQuery);
nodesDestroyNode((SNode*)stmt);
return TSDB_CODE_OUT_OF_MEMORY;
}
stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
stmt->freeArrayFunc = insDestroyVgroupDataCxtList; stmt->freeArrayFunc = insDestroyVgroupDataCxtList;
pQuery->pRoot = (SNode*)stmt; pQuery->pRoot = (SNode*)stmt;
return pQuery; *query = pQuery;
return TSDB_CODE_SUCCESS;
} }
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {

1608
source/libs/parser/src/parTranslater.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/qcom/src/queryUtil.c
View File

@ -158,7 +158,9 @@ int32_t cleanupTaskQueue() {
} }
int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) {
SSchedMsg* pSchedMsg = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0); SSchedMsg* pSchedMsg;
int32_t rc = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0, (void **)&pSchedMsg);
if (rc) return rc;
pSchedMsg->fp = NULL; pSchedMsg->fp = NULL;
pSchedMsg->ahandle = execFn; pSchedMsg->ahandle = execFn;
pSchedMsg->thandle = execParam; pSchedMsg->thandle = execParam;

2
source/libs/stream/src/streamCheckStatus.c
View File

@ -310,7 +310,7 @@ void streamTaskCleanupCheckInfo(STaskCheckInfo* pInfo) {
taosArrayDestroy(pInfo->pList); taosArrayDestroy(pInfo->pList);
pInfo->pList = NULL; pInfo->pList = NULL;
if (pInfo->checkRspTmr != NULL) { if (pInfo->checkRspTmr != NULL) {
/*bool ret = */ taosTmrStop(pInfo->checkRspTmr); /*bool ret = */ taosTmrStop(pInfo->checkRspTmr);
pInfo->checkRspTmr = NULL; pInfo->checkRspTmr = NULL;

Some files were not shown because too many files have changed in this diff Show More