xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

merge 3.0

This commit is contained in:
Xiaoyu Wang 2022-12-23 17:50:05 +08:00
parent 35b779c128 401396a0b9
commit 0c269c317b
119 changed files with 10923 additions and 4605 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -173,7 +173,7 @@ def pre_test_build_mac() {
''' '''
sh ''' sh '''
cd ${WK}/debug cd ${WK}/debug
cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release
make -j10 make -j10
ctest -j10 || exit 7 ctest -j10 || exit 7
''' '''

4
cmake/cmake.define
View File

@ -124,7 +124,9 @@ ELSE ()
ENDIF () ENDIF ()
INCLUDE(CheckCCompilerFlag) INCLUDE(CheckCCompilerFlag)
IF (("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") OR ("${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")) IF (TD_ARM_64 OR TD_ARM_32)
SET(COMPILER_SUPPORT_SSE42 false)
ELSEIF (("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") OR ("${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang"))
SET(COMPILER_SUPPORT_SSE42 true) SET(COMPILER_SUPPORT_SSE42 true)
MESSAGE(STATUS "Always enable sse4.2 for Clang/AppleClang") MESSAGE(STATUS "Always enable sse4.2 for Clang/AppleClang")
ELSE() ELSE()

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter # taosadapter
ExternalProject_Add(taosadapter ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG 566540d GIT_TAG f0c1753
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG b20c9d1 GIT_TAG 261fcca
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

3
docs/en/12-taos-sql/10-function.md
View File

@ -876,7 +876,8 @@ INTERP(expr)
- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. The parameter `EVERY` must be an integer, with no quotes, with a time unit of: b(nanosecond), u(microsecond), a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds.
- Interpolation is performed based on `FILL` parameter. - Interpolation is performed based on `FILL` parameter.
- `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable.
- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). - Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4).
- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.2.1).
### LAST ### LAST

2
docs/en/12-taos-sql/14-stream.md
View File

@ -108,7 +108,7 @@ SHOW STREAMS;
When you create a stream, you can use the TRIGGER parameter to specify triggering conditions for it. When you create a stream, you can use the TRIGGER parameter to specify triggering conditions for it.
For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering: For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering，the default value is AT_ONCE:
1. AT_ONCE: triggers on write 1. AT_ONCE: triggers on write

71
docs/en/12-taos-sql/24-show.md
View File

@ -178,6 +178,77 @@ SHOW TABLE DISTRIBUTED table_name;
Shows how table data is distributed. Shows how table data is distributed.
Examples： show table distributed d0\G; Display the block distribution of table `d0` in detailed format.
*************************** 1.row ***************************
_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
Total_Blocks : Table `d0` contains total 5 blocks
Total_Size: The total size of all the data blocks in table `d0` is 93.65 KB
Average_size: The average size of each block is 18.73 KB
Compression_Ratio: The data compression rate is 23.98%
*************************** 2.row ***************************
_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
Total_Rows: Table `d0` contains 20,000 rows
Inmem_Rows： The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows
MinRows： The minimum number of rows in a block is 3,616
MaxRows： The maximum number of rows in a block is 4,096B
Average_Rows： The average number of rows in a block is 4,000
*************************** 3.row ***************************
_block_dist: Total_Tables=[1] Total_Files=[2]
Total_Tables: The number of child tables, 1 in this example
Total_Files： The number of files storing the table's data, 2 in this example
*************************** 4.row ***************************
_block_dist: --------------------------------------------------------------------------------
*************************** 5.row ***************************
_block_dist: 0100 |
*************************** 6.row ***************************
_block_dist: 0299 |
*************************** 7.row ***************************
_block_dist: 0498 |
*************************** 8.row ***************************
_block_dist: 0697 |
*************************** 9.row ***************************
_block_dist: 0896 |
*************************** 10.row ***************************
_block_dist: 1095 |
*************************** 11.row ***************************
_block_dist: 1294 |
*************************** 12.row ***************************
_block_dist: 1493 |
*************************** 13.row ***************************
_block_dist: 1692 |
*************************** 14.row ***************************
_block_dist: 1891 |
*************************** 15.row ***************************
_block_dist: 2090 |
*************************** 16.row ***************************
_block_dist: 2289 |
*************************** 17.row ***************************
_block_dist: 2488 |
*************************** 18.row ***************************
_block_dist: 2687 |
*************************** 19.row ***************************
_block_dist: 2886 |
*************************** 20.row ***************************
_block_dist: 3085 |
*************************** 21.row ***************************
_block_dist: 3284 |
*************************** 22.row ***************************
_block_dist: 3483 ||||||||||||||||| 1 (20.00%)
*************************** 23.row ***************************
_block_dist: 3682 |
*************************** 24.row ***************************
_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)
Query OK, 24 row(s) in set (0.002444s)
The above show the block distribution percentage according to the number of rows in each block. In the above example, `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. The number of blocks whose rows fall in other range is zero.
## SHOW TAGS ## SHOW TAGS
```sql ```sql

4
docs/en/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.2.1
<Release type="tdengine" version="3.0.2.1" />
## 3.0.2.0 ## 3.0.2.0
<Release type="tdengine" version="3.0.2.0" /> <Release type="tdengine" version="3.0.2.0" />

4
docs/en/28-releases/02-tools.md
View File

@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.3.3
<Release type="tools" version="2.3.3" />
## 2.3.2 ## 2.3.2
<Release type="tools" version="2.3.2" /> <Release type="tools" version="2.3.2" />

2
docs/examples/go/sub/main.go
View File

@ -24,7 +24,7 @@ func main() {
if err != nil { if err != nil {
panic(err) panic(err)
} }
_, err = db.Exec("create topic if not exists example_tmq_topic with meta as DATABASE example_tmq") _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq")
if err != nil { if err != nil {
panic(err) panic(err)
} }

1
docs/zh/08-connector/05-schemaless-api.mdx
View File

@ -19,6 +19,7 @@ TDengine 提供了兼容 InfluxDB (v1) 和 OpenTSDB 行协议的 Schemaless API
- `precision` TDengine 使用的时间精度 - `precision` TDengine 使用的时间精度
- `u` TDengine 用户名 - `u` TDengine 用户名
- `p` TDengine 密码 - `p` TDengine 密码
- `ttl` 自动创建的子表生命周期，以子表的第一条数据的 TTL 参数为准，不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数
注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。 注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。

4
docs/zh/12-taos-sql/04-stable.md
View File

@ -139,10 +139,10 @@ alter_table_option: {
- ADD COLUMN：添加列。 - ADD COLUMN：添加列。
- DROP COLUMN：删除列。 - DROP COLUMN：删除列。
- MODIFY COLUMN：修改列定义，如果数据列的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 - MODIFY COLUMN：修改列的宽度，数据列的类型必须是 nchar 和 binary，使用此指令可以修改其宽度，只能改大，不能改小。
- ADD TAG：给超级表添加一个标签。 - ADD TAG：给超级表添加一个标签。
- DROP TAG：删除超级表的一个标签。从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。 - DROP TAG：删除超级表的一个标签。从超级表删除某个标签后，该超级表下的所有子表也会自动删除该标签。
- MODIFY TAG：修改超级表的一个标签的定义。如果标签的类型是可变长类型，那么可以使用此指令修改其宽度，只能改大，不能改小。 - MODIFY TAG：修改超级表的一个标签的列宽度。标签的类型只能是 nchar 和 binary，使用此指令可以修改其宽度，只能改大，不能改小。
- RENAME TAG：修改超级表的一个标签的名称。从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。 - RENAME TAG：修改超级表的一个标签的名称。从超级表修改某个标签名后，该超级表下的所有子表也会自动更新该标签名。
### 增加列 ### 增加列

1
docs/zh/12-taos-sql/10-function.md
View File

@ -880,6 +880,7 @@ INTERP(expr)
- INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。
- INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。 - INTERP 只能在一个时间序列内进行插值，因此当作用于超级表时必须跟 partition by tbname 一起使用。
- INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 - INTERP 可以与伪列 _irowts 一起使用，返回插值点所对应的时间戳(3.0.1.4版本以后支持)。
- INTERP 可以与伪列 _isfilled 一起使用，显示返回结果是否为原始记录或插值算法产生的数据(3.0.2.1版本以后支持)。
### LAST ### LAST

2
docs/zh/12-taos-sql/14-stream.md
View File

@ -114,7 +114,7 @@ SELECT * from information_schema.`ins_streams`;
在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。 在创建流时，可以通过 TRIGGER 指令指定流式计算的触发模式。
对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式： 对于非窗口计算，流式计算的触发是实时的；对于窗口计算，目前提供 3 种触发模式，默认为 AT_ONCE：
1. AT_ONCE：写入立即触发 1. AT_ONCE：写入立即触发

69
docs/zh/12-taos-sql/24-show.md
View File

@ -179,6 +179,75 @@ SHOW TABLE DISTRIBUTED table_name;
显示表的数据分布信息。 显示表的数据分布信息。
示例说明：
语句： show table distributed d0\G; 竖行显示表 d0 的 BLOCK 分布情况
*************************** 1.row ***************************
_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
Total_Blocks : 表d0 占用的 block 个数为 5 个
Total_Size. : 表 d0 所有 block 在文件中占用的大小为 93.65 KB
Average_size: 平均每个 block 在文件中占用的空间大小为 18.73 KB
Compression_Ratio: 数据压缩率为 23.98%
*************************** 2.row ***************************
_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
Total_Rows: 统计表 d0 的所有行数 为20000 行
Inmem_Rows： 表示仍然还存放在内存中的行数，即没有落盘的行数，为 0行，表示没有
MinRows： BLOCK 中最小的行数，为 3616 行
MaxRows： BLOCK 中最大的行数，为 4096行
Average_Rows： BLOCK 中的平均行数，为4000 行
*************************** 3.row ***************************
_block_dist: Total_Tables=[1] Total_Files=[2]
Total_Tables: 表示子表的个数，这里为1
Total_Files： 表数据保存在几个文件中，这里保存在 2 个文件中
*************************** 5.row ***************************
_block_dist: 0100 |
*************************** 6.row ***************************
_block_dist: 0299 |
......
*************************** 22.row ***************************
_block_dist: 3483 ||||||||||||||||| 1 (20.00%)
*************************** 23.row ***************************
_block_dist: 3682 |
*************************** 24.row ***************************
_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)
Query OK, 24 row(s) in set (0.002444s)
上面是块中包含数据行数的块儿分布情况图，这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数，上面的意思就是这个表的 5 个块，分布在 3483 ~3681 行的块有 1 个，占整个块的 20%，分布在 3881 ~ 4096（最大行数）的块数为 4 个，占整个块的 80%， 其它区域内分布块数为 0。
## SHOW TAGS ## SHOW TAGS
```sql ```sql

4
docs/zh/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.0.2.1
<Release type="tdengine" version="3.0.2.1" />
## 3.0.2.0 ## 3.0.2.0
<Release type="tdengine" version="3.0.2.0" /> <Release type="tdengine" version="3.0.2.0" />

4
docs/zh/28-releases/02-tools.md
View File

@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下：
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 2.3.3
<Release type="tools" version="2.3.3" />
## 2.3.2 ## 2.3.2
<Release type="tools" version="2.3.2" /> <Release type="tools" version="2.3.2" />

2
examples/c/asyncdemo.c
View File

@ -35,7 +35,7 @@ int64_t st, et;
typedef struct { typedef struct {
int id; int id;
TAOS *taos; TAOS *taos;
char name[16]; char name[32];
time_t timeStamp; time_t timeStamp;
int value; int value;
int rowsInserted; int rowsInserted;

34
examples/c/tmq.c
View File

@ -20,7 +20,7 @@
#include <time.h> #include <time.h>
#include "taos.h" #include "taos.h"
static int running = 1; static int running = 1;
static int32_t msg_process(TAOS_RES* msg) { static int32_t msg_process(TAOS_RES* msg) {
char buf[1024]; char buf[1024];
@ -40,8 +40,8 @@ static int32_t msg_process(TAOS_RES* msg) {
TAOS_FIELD* fields = taos_fetch_fields(msg); TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg); int32_t numOfFields = taos_field_count(msg);
//int32_t* length = taos_fetch_lengths(msg); // int32_t* length = taos_fetch_lengths(msg);
int32_t precision = taos_result_precision(msg); int32_t precision = taos_result_precision(msg);
rows++; rows++;
taos_print_row(buf, row, fields, numOfFields); taos_print_row(buf, row, fields, numOfFields);
printf("precision: %d, row content: %s\n", precision, buf); printf("precision: %d, row content: %s\n", precision, buf);
@ -62,14 +62,12 @@ static int32_t init_env() {
pRes = taos_query(pConn, "drop topic topicname"); pRes = taos_query(pConn, "drop topic topicname");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes));
return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "drop database if exists tmqdb"); pRes = taos_query(pConn, "drop database if exists tmqdb");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes));
return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -77,7 +75,7 @@ static int32_t init_env() {
pRes = taos_query(pConn, "create database tmqdb precision 'ns'"); pRes = taos_query(pConn, "create database tmqdb precision 'ns'");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -87,7 +85,7 @@ static int32_t init_env() {
pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -96,28 +94,28 @@ static int32_t init_env() {
pRes = taos_query(pConn, "create table tmqdb.ctb0 using tmqdb.stb tags(0, 'subtable0')"); pRes = taos_query(pConn, "create table tmqdb.ctb0 using tmqdb.stb tags(0, 'subtable0')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table ctb0, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table ctb0, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table tmqdb.ctb1 using tmqdb.stb tags(1, 'subtable1')"); pRes = taos_query(pConn, "create table tmqdb.ctb1 using tmqdb.stb tags(1, 'subtable1')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table ctb1, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table ctb1, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table tmqdb.ctb2 using tmqdb.stb tags(2, 'subtable2')"); pRes = taos_query(pConn, "create table tmqdb.ctb2 using tmqdb.stb tags(2, 'subtable2')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table ctb2, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table ctb2, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table tmqdb.ctb3 using tmqdb.stb tags(3, 'subtable3')"); pRes = taos_query(pConn, "create table tmqdb.ctb3 using tmqdb.stb tags(3, 'subtable3')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create super table ctb3, reason:%s\n", taos_errstr(pRes)); printf("failed to create super table ctb3, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -126,33 +124,37 @@ static int32_t init_env() {
pRes = taos_query(pConn, "insert into tmqdb.ctb0 values(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00')"); pRes = taos_query(pConn, "insert into tmqdb.ctb0 values(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tmqdb.ctb1 values(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11')"); pRes = taos_query(pConn, "insert into tmqdb.ctb1 values(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22')"); pRes = taos_query(pConn, "insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33')"); pRes = taos_query(pConn, "insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33')");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes));
return -1; goto END;
} }
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
return 0; return 0;
END:
taos_free_result(pRes);
taos_close(pConn);
return -1;
} }
int32_t create_topic() { int32_t create_topic() {

15
include/client/taos.h
View File

@ -59,6 +59,7 @@ typedef enum {
TSDB_OPTION_TIMEZONE, TSDB_OPTION_TIMEZONE,
TSDB_OPTION_CONFIGDIR, TSDB_OPTION_CONFIGDIR,
TSDB_OPTION_SHELL_ACTIVITY_TIMER, TSDB_OPTION_SHELL_ACTIVITY_TIMER,
TSDB_OPTION_USE_ADAPTER,
TSDB_MAX_OPTIONS TSDB_MAX_OPTIONS
} TSDB_OPTION; } TSDB_OPTION;
@ -218,7 +219,7 @@ DLL_EXPORT const void *taos_get_raw_block(TAOS_RES *res);
DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_INFO *dbInfo); DLL_EXPORT int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_INFO *dbInfo);
DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId); DLL_EXPORT int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
/* --------------------------schemaless INTERFACE------------------------------- */ /* --------------------------schemaless INTERFACE------------------------------- */
@ -229,13 +230,14 @@ DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len
int precision); int precision);
DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows,
int protocol, int precision, int64_t reqid); int protocol, int precision, int64_t reqid);
DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl); DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
int32_t ttl);
DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, DLL_EXPORT TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol,
int precision, int32_t ttl, int64_t reqid); int precision, int32_t ttl, int64_t reqid);
DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl); int precision, int32_t ttl);
DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, DLL_EXPORT TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows,
int protocol, int precision, int32_t ttl, int64_t reqid); int protocol, int precision, int32_t ttl, int64_t reqid);
/* --------------------------TMQ INTERFACE------------------------------- */ /* --------------------------TMQ INTERFACE------------------------------- */
@ -308,7 +310,8 @@ DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw); DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw); DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw);
DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname); DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
DLL_EXPORT int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields); DLL_EXPORT int taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields);
DLL_EXPORT void tmq_free_raw(tmq_raw_data raw); DLL_EXPORT void tmq_free_raw(tmq_raw_data raw);
// Returning null means error. Returned result need to be freed by tmq_free_json_meta // Returning null means error. Returned result need to be freed by tmq_free_json_meta
DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res); DLL_EXPORT char *tmq_get_json_meta(TAOS_RES *res);

19
include/common/tcommon.h
View File

@ -195,7 +195,7 @@ typedef struct SDataBlockInfo {
uint32_t capacity; uint32_t capacity;
SBlockID id; SBlockID id;
int16_t hasVarCol; int16_t hasVarCol;
int16_t dataLoad; // denote if the data is loaded or not int16_t dataLoad; // denote if the data is loaded or not
// TODO: optimize and remove following // TODO: optimize and remove following
int64_t version; // used for stream, and need serialization int64_t version; // used for stream, and need serialization
@ -204,8 +204,9 @@ typedef struct SDataBlockInfo {
STimeWindow calWin; // used for stream, do not serialize STimeWindow calWin; // used for stream, do not serialize
TSKEY watermark; // used for stream TSKEY watermark; // used for stream
char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition
STag* pTag; // used for stream partition int32_t tagLen;
void* pTag; // used for stream partition
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SSDataBlock { typedef struct SSDataBlock {
@ -239,22 +240,22 @@ typedef struct SVarColAttr {
// pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == info.rows, all data are null
// pBlockAgg->numOfNull == 0, no data are null. // pBlockAgg->numOfNull == 0, no data are null.
typedef struct SColumnInfoData { typedef struct SColumnInfoData {
char* pData; // the corresponding block data in memory char* pData; // the corresponding block data in memory
union { union {
char* nullbitmap; // bitmap, one bit for each item in the list char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta; SVarColAttr varmeta;
}; };
SColumnInfo info; // column info SColumnInfo info; // column info
bool hasNull; // if current column data has null value. bool hasNull; // if current column data has null value.
} SColumnInfoData; } SColumnInfoData;
typedef struct SQueryTableDataCond { typedef struct SQueryTableDataCond {
uint64_t suid; uint64_t suid;
int32_t order; // desc|asc order to iterate the data block int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols; int32_t numOfCols;
SColumnInfo* colList; SColumnInfo* colList;
int32_t* pSlotList; // the column output destation slot, and it may be null int32_t* pSlotList; // the column output destation slot, and it may be null
int32_t type; // data block load type: int32_t type; // data block load type:
STimeWindow twindows; STimeWindow twindows;
int64_t startVersion; int64_t startVersion;
int64_t endVersion; int64_t endVersion;

1
include/common/tglobal.h
View File

@ -101,6 +101,7 @@ extern int32_t tsRedirectPeriod;
extern int32_t tsRedirectFactor; extern int32_t tsRedirectFactor;
extern int32_t tsRedirectMaxPeriod; extern int32_t tsRedirectMaxPeriod;
extern int32_t tsMaxRetryWaitTime; extern int32_t tsMaxRetryWaitTime;
extern bool tsUseAdapter;
// client // client
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;

647
include/common/ttokendef.h
View File

@ -16,329 +16,330 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_USER 33 #define TK_USER 33
#define TK_ENABLE 34 #define TK_ENABLE 34
#define TK_NK_INTEGER 35 #define TK_NK_INTEGER 35
#define TK_SYSINFO 36 #define TK_SYSINFO 36
#define TK_DROP 37 #define TK_DROP 37
#define TK_GRANT 38 #define TK_GRANT 38
#define TK_ON 39 #define TK_ON 39
#define TK_TO 40 #define TK_TO 40
#define TK_REVOKE 41 #define TK_REVOKE 41
#define TK_FROM 42 #define TK_FROM 42
#define TK_SUBSCRIBE 43 #define TK_SUBSCRIBE 43
#define TK_NK_COMMA 44 #define TK_NK_COMMA 44
#define TK_READ 45 #define TK_READ 45
#define TK_WRITE 46 #define TK_WRITE 46
#define TK_NK_DOT 47 #define TK_NK_DOT 47
#define TK_DNODE 48 #define TK_DNODE 48
#define TK_PORT 49 #define TK_PORT 49
#define TK_DNODES 50 #define TK_DNODES 50
#define TK_NK_IPTOKEN 51 #define TK_NK_IPTOKEN 51
#define TK_FORCE 52 #define TK_FORCE 52
#define TK_LOCAL 53 #define TK_LOCAL 53
#define TK_QNODE 54 #define TK_QNODE 54
#define TK_BNODE 55 #define TK_BNODE 55
#define TK_SNODE 56 #define TK_SNODE 56
#define TK_MNODE 57 #define TK_MNODE 57
#define TK_DATABASE 58 #define TK_DATABASE 58
#define TK_USE 59 #define TK_USE 59
#define TK_FLUSH 60 #define TK_FLUSH 60
#define TK_TRIM 61 #define TK_TRIM 61
#define TK_IF 62 #define TK_IF 62
#define TK_NOT 63 #define TK_NOT 63
#define TK_EXISTS 64 #define TK_EXISTS 64
#define TK_BUFFER 65 #define TK_BUFFER 65
#define TK_CACHEMODEL 66 #define TK_CACHEMODEL 66
#define TK_CACHESIZE 67 #define TK_CACHESIZE 67
#define TK_COMP 68 #define TK_COMP 68
#define TK_DURATION 69 #define TK_DURATION 69
#define TK_NK_VARIABLE 70 #define TK_NK_VARIABLE 70
#define TK_MAXROWS 71 #define TK_MAXROWS 71
#define TK_MINROWS 72 #define TK_MINROWS 72
#define TK_KEEP 73 #define TK_KEEP 73
#define TK_PAGES 74 #define TK_PAGES 74
#define TK_PAGESIZE 75 #define TK_PAGESIZE 75
#define TK_TSDB_PAGESIZE 76 #define TK_TSDB_PAGESIZE 76
#define TK_PRECISION 77 #define TK_PRECISION 77
#define TK_REPLICA 78 #define TK_REPLICA 78
#define TK_VGROUPS 79 #define TK_VGROUPS 79
#define TK_SINGLE_STABLE 80 #define TK_SINGLE_STABLE 80
#define TK_RETENTIONS 81 #define TK_RETENTIONS 81
#define TK_SCHEMALESS 82 #define TK_SCHEMALESS 82
#define TK_WAL_LEVEL 83 #define TK_WAL_LEVEL 83
#define TK_WAL_FSYNC_PERIOD 84 #define TK_WAL_FSYNC_PERIOD 84
#define TK_WAL_RETENTION_PERIOD 85 #define TK_WAL_RETENTION_PERIOD 85
#define TK_WAL_RETENTION_SIZE 86 #define TK_WAL_RETENTION_SIZE 86
#define TK_WAL_ROLL_PERIOD 87 #define TK_WAL_ROLL_PERIOD 87
#define TK_WAL_SEGMENT_SIZE 88 #define TK_WAL_SEGMENT_SIZE 88
#define TK_STT_TRIGGER 89 #define TK_STT_TRIGGER 89
#define TK_TABLE_PREFIX 90 #define TK_TABLE_PREFIX 90
#define TK_TABLE_SUFFIX 91 #define TK_TABLE_SUFFIX 91
#define TK_NK_COLON 92 #define TK_NK_COLON 92
#define TK_MAX_SPEED 93 #define TK_MAX_SPEED 93
#define TK_TABLE 94 #define TK_TABLE 94
#define TK_NK_LP 95 #define TK_NK_LP 95
#define TK_NK_RP 96 #define TK_NK_RP 96
#define TK_STABLE 97 #define TK_STABLE 97
#define TK_ADD 98 #define TK_ADD 98
#define TK_COLUMN 99 #define TK_COLUMN 99
#define TK_MODIFY 100 #define TK_MODIFY 100
#define TK_RENAME 101 #define TK_RENAME 101
#define TK_TAG 102 #define TK_TAG 102
#define TK_SET 103 #define TK_SET 103
#define TK_NK_EQ 104 #define TK_NK_EQ 104
#define TK_USING 105 #define TK_USING 105
#define TK_TAGS 106 #define TK_TAGS 106
#define TK_COMMENT 107 #define TK_COMMENT 107
#define TK_BOOL 108 #define TK_BOOL 108
#define TK_TINYINT 109 #define TK_TINYINT 109
#define TK_SMALLINT 110 #define TK_SMALLINT 110
#define TK_INT 111 #define TK_INT 111
#define TK_INTEGER 112 #define TK_INTEGER 112
#define TK_BIGINT 113 #define TK_BIGINT 113
#define TK_FLOAT 114 #define TK_FLOAT 114
#define TK_DOUBLE 115 #define TK_DOUBLE 115
#define TK_BINARY 116 #define TK_BINARY 116
#define TK_TIMESTAMP 117 #define TK_TIMESTAMP 117
#define TK_NCHAR 118 #define TK_NCHAR 118
#define TK_UNSIGNED 119 #define TK_UNSIGNED 119
#define TK_JSON 120 #define TK_JSON 120
#define TK_VARCHAR 121 #define TK_VARCHAR 121
#define TK_MEDIUMBLOB 122 #define TK_MEDIUMBLOB 122
#define TK_BLOB 123 #define TK_BLOB 123
#define TK_VARBINARY 124 #define TK_VARBINARY 124
#define TK_DECIMAL 125 #define TK_DECIMAL 125
#define TK_MAX_DELAY 126 #define TK_MAX_DELAY 126
#define TK_WATERMARK 127 #define TK_WATERMARK 127
#define TK_ROLLUP 128 #define TK_ROLLUP 128
#define TK_TTL 129 #define TK_TTL 129
#define TK_SMA 130 #define TK_SMA 130
#define TK_DELETE_MARK 131 #define TK_DELETE_MARK 131
#define TK_FIRST 132 #define TK_FIRST 132
#define TK_LAST 133 #define TK_LAST 133
#define TK_SHOW 134 #define TK_SHOW 134
#define TK_PRIVILEGES 135 #define TK_PRIVILEGES 135
#define TK_DATABASES 136 #define TK_DATABASES 136
#define TK_TABLES 137 #define TK_TABLES 137
#define TK_STABLES 138 #define TK_STABLES 138
#define TK_MNODES 139 #define TK_MNODES 139
#define TK_QNODES 140 #define TK_QNODES 140
#define TK_FUNCTIONS 141 #define TK_FUNCTIONS 141
#define TK_INDEXES 142 #define TK_INDEXES 142
#define TK_ACCOUNTS 143 #define TK_ACCOUNTS 143
#define TK_APPS 144 #define TK_APPS 144
#define TK_CONNECTIONS 145 #define TK_CONNECTIONS 145
#define TK_LICENCES 146 #define TK_LICENCES 146
#define TK_GRANTS 147 #define TK_GRANTS 147
#define TK_QUERIES 148 #define TK_QUERIES 148
#define TK_SCORES 149 #define TK_SCORES 149
#define TK_TOPICS 150 #define TK_TOPICS 150
#define TK_VARIABLES 151 #define TK_VARIABLES 151
#define TK_CLUSTER 152 #define TK_CLUSTER 152
#define TK_BNODES 153 #define TK_BNODES 153
#define TK_SNODES 154 #define TK_SNODES 154
#define TK_TRANSACTIONS 155 #define TK_TRANSACTIONS 155
#define TK_DISTRIBUTED 156 #define TK_DISTRIBUTED 156
#define TK_CONSUMERS 157 #define TK_CONSUMERS 157
#define TK_SUBSCRIPTIONS 158 #define TK_SUBSCRIPTIONS 158
#define TK_VNODES 159 #define TK_VNODES 159
#define TK_LIKE 160 #define TK_LIKE 160
#define TK_TBNAME 161 #define TK_TBNAME 161
#define TK_QTAGS 162 #define TK_QTAGS 162
#define TK_AS 163 #define TK_AS 163
#define TK_INDEX 164 #define TK_INDEX 164
#define TK_FUNCTION 165 #define TK_FUNCTION 165
#define TK_INTERVAL 166 #define TK_INTERVAL 166
#define TK_TOPIC 167 #define TK_COUNT 167
#define TK_WITH 168 #define TK_LAST_ROW 168
#define TK_META 169 #define TK_TOPIC 169
#define TK_CONSUMER 170 #define TK_WITH 170
#define TK_GROUP 171 #define TK_META 171
#define TK_DESC 172 #define TK_CONSUMER 172
#define TK_DESCRIBE 173 #define TK_GROUP 173
#define TK_RESET 174 #define TK_DESC 174
#define TK_QUERY 175 #define TK_DESCRIBE 175
#define TK_CACHE 176 #define TK_RESET 176
#define TK_EXPLAIN 177 #define TK_QUERY 177
#define TK_ANALYZE 178 #define TK_CACHE 178
#define TK_VERBOSE 179 #define TK_EXPLAIN 179
#define TK_NK_BOOL 180 #define TK_ANALYZE 180
#define TK_RATIO 181 #define TK_VERBOSE 181
#define TK_NK_FLOAT 182 #define TK_NK_BOOL 182
#define TK_OUTPUTTYPE 183 #define TK_RATIO 183
#define TK_AGGREGATE 184 #define TK_NK_FLOAT 184
#define TK_BUFSIZE 185 #define TK_OUTPUTTYPE 185
#define TK_STREAM 186 #define TK_AGGREGATE 186
#define TK_INTO 187 #define TK_BUFSIZE 187
#define TK_TRIGGER 188 #define TK_STREAM 188
#define TK_AT_ONCE 189 #define TK_INTO 189
#define TK_WINDOW_CLOSE 190 #define TK_TRIGGER 190
#define TK_IGNORE 191 #define TK_AT_ONCE 191
#define TK_EXPIRED 192 #define TK_WINDOW_CLOSE 192
#define TK_FILL_HISTORY 193 #define TK_IGNORE 193
#define TK_SUBTABLE 194 #define TK_EXPIRED 194
#define TK_KILL 195 #define TK_FILL_HISTORY 195
#define TK_CONNECTION 196 #define TK_SUBTABLE 196
#define TK_TRANSACTION 197 #define TK_KILL 197
#define TK_BALANCE 198 #define TK_CONNECTION 198
#define TK_VGROUP 199 #define TK_TRANSACTION 199
#define TK_MERGE 200 #define TK_BALANCE 200
#define TK_REDISTRIBUTE 201 #define TK_VGROUP 201
#define TK_SPLIT 202 #define TK_MERGE 202
#define TK_DELETE 203 #define TK_REDISTRIBUTE 203
#define TK_INSERT 204 #define TK_SPLIT 204
#define TK_NULL 205 #define TK_DELETE 205
#define TK_NK_QUESTION 206 #define TK_INSERT 206
#define TK_NK_ARROW 207 #define TK_NULL 207
#define TK_ROWTS 208 #define TK_NK_QUESTION 208
#define TK_QSTART 209 #define TK_NK_ARROW 209
#define TK_QEND 210 #define TK_ROWTS 210
#define TK_QDURATION 211 #define TK_QSTART 211
#define TK_WSTART 212 #define TK_QEND 212
#define TK_WEND 213 #define TK_QDURATION 213
#define TK_WDURATION 214 #define TK_WSTART 214
#define TK_IROWTS 215 #define TK_WEND 215
#define TK_CAST 216 #define TK_WDURATION 216
#define TK_NOW 217 #define TK_IROWTS 217
#define TK_TODAY 218 #define TK_ISFILLED 218
#define TK_TIMEZONE 219 #define TK_CAST 219
#define TK_CLIENT_VERSION 220 #define TK_NOW 220
#define TK_SERVER_VERSION 221 #define TK_TODAY 221
#define TK_SERVER_STATUS 222 #define TK_TIMEZONE 222
#define TK_CURRENT_USER 223 #define TK_CLIENT_VERSION 223
#define TK_COUNT 224 #define TK_SERVER_VERSION 224
#define TK_LAST_ROW 225 #define TK_SERVER_STATUS 225
#define TK_CASE 226 #define TK_CURRENT_USER 226
#define TK_END 227 #define TK_CASE 227
#define TK_WHEN 228 #define TK_END 228
#define TK_THEN 229 #define TK_WHEN 229
#define TK_ELSE 230 #define TK_THEN 230
#define TK_BETWEEN 231 #define TK_ELSE 231
#define TK_IS 232 #define TK_BETWEEN 232
#define TK_NK_LT 233 #define TK_IS 233
#define TK_NK_GT 234 #define TK_NK_LT 234
#define TK_NK_LE 235 #define TK_NK_GT 235
#define TK_NK_GE 236 #define TK_NK_LE 236
#define TK_NK_NE 237 #define TK_NK_GE 237
#define TK_MATCH 238 #define TK_NK_NE 238
#define TK_NMATCH 239 #define TK_MATCH 239
#define TK_CONTAINS 240 #define TK_NMATCH 240
#define TK_IN 241 #define TK_CONTAINS 241
#define TK_JOIN 242 #define TK_IN 242
#define TK_INNER 243 #define TK_JOIN 243
#define TK_SELECT 244 #define TK_INNER 244
#define TK_DISTINCT 245 #define TK_SELECT 245
#define TK_WHERE 246 #define TK_DISTINCT 246
#define TK_PARTITION 247 #define TK_WHERE 247
#define TK_BY 248 #define TK_PARTITION 248
#define TK_SESSION 249 #define TK_BY 249
#define TK_STATE_WINDOW 250 #define TK_SESSION 250
#define TK_EVENT_WINDOW 251 #define TK_STATE_WINDOW 251
#define TK_START 252 #define TK_EVENT_WINDOW 252
#define TK_SLIDING 253 #define TK_START 253
#define TK_FILL 254 #define TK_SLIDING 254
#define TK_VALUE 255 #define TK_FILL 255
#define TK_NONE 256 #define TK_VALUE 256
#define TK_PREV 257 #define TK_NONE 257
#define TK_LINEAR 258 #define TK_PREV 258
#define TK_NEXT 259 #define TK_LINEAR 259
#define TK_HAVING 260 #define TK_NEXT 260
#define TK_RANGE 261 #define TK_HAVING 261
#define TK_EVERY 262 #define TK_RANGE 262
#define TK_ORDER 263 #define TK_EVERY 263
#define TK_SLIMIT 264 #define TK_ORDER 264
#define TK_SOFFSET 265 #define TK_SLIMIT 265
#define TK_LIMIT 266 #define TK_SOFFSET 266
#define TK_OFFSET 267 #define TK_LIMIT 267
#define TK_ASC 268 #define TK_OFFSET 268
#define TK_NULLS 269 #define TK_ASC 269
#define TK_ABORT 270 #define TK_NULLS 270
#define TK_AFTER 271 #define TK_ABORT 271
#define TK_ATTACH 272 #define TK_AFTER 272
#define TK_BEFORE 273 #define TK_ATTACH 273
#define TK_BEGIN 274 #define TK_BEFORE 274
#define TK_BITAND 275 #define TK_BEGIN 275
#define TK_BITNOT 276 #define TK_BITAND 276
#define TK_BITOR 277 #define TK_BITNOT 277
#define TK_BLOCKS 278 #define TK_BITOR 278
#define TK_CHANGE 279 #define TK_BLOCKS 279
#define TK_COMMA 280 #define TK_CHANGE 280
#define TK_COMPACT 281 #define TK_COMMA 281
#define TK_CONCAT 282 #define TK_COMPACT 282
#define TK_CONFLICT 283 #define TK_CONCAT 283
#define TK_COPY 284 #define TK_CONFLICT 284
#define TK_DEFERRED 285 #define TK_COPY 285
#define TK_DELIMITERS 286 #define TK_DEFERRED 286
#define TK_DETACH 287 #define TK_DELIMITERS 287
#define TK_DIVIDE 288 #define TK_DETACH 288
#define TK_DOT 289 #define TK_DIVIDE 289
#define TK_EACH 290 #define TK_DOT 290
#define TK_FAIL 291 #define TK_EACH 291
#define TK_FILE 292 #define TK_FAIL 292
#define TK_FOR 293 #define TK_FILE 293
#define TK_GLOB 294 #define TK_FOR 294
#define TK_ID 295 #define TK_GLOB 295
#define TK_IMMEDIATE 296 #define TK_ID 296
#define TK_IMPORT 297 #define TK_IMMEDIATE 297
#define TK_INITIALLY 298 #define TK_IMPORT 298
#define TK_INSTEAD 299 #define TK_INITIALLY 299
#define TK_ISNULL 300 #define TK_INSTEAD 300
#define TK_KEY 301 #define TK_ISNULL 301
#define TK_MODULES 302 #define TK_KEY 302
#define TK_NK_BITNOT 303 #define TK_MODULES 303
#define TK_NK_SEMI 304 #define TK_NK_BITNOT 304
#define TK_NOTNULL 305 #define TK_NK_SEMI 305
#define TK_OF 306 #define TK_NOTNULL 306
#define TK_PLUS 307 #define TK_OF 307
#define TK_PRIVILEGE 308 #define TK_PLUS 308
#define TK_RAISE 309 #define TK_PRIVILEGE 309
#define TK_REPLACE 310 #define TK_RAISE 310
#define TK_RESTRICT 311 #define TK_REPLACE 311
#define TK_ROW 312 #define TK_RESTRICT 312
#define TK_SEMI 313 #define TK_ROW 313
#define TK_STAR 314 #define TK_SEMI 314
#define TK_STATEMENT 315 #define TK_STAR 315
#define TK_STRICT 316 #define TK_STATEMENT 316
#define TK_STRING 317 #define TK_STRICT 317
#define TK_TIMES 318 #define TK_STRING 318
#define TK_UPDATE 319 #define TK_TIMES 319
#define TK_VALUES 320 #define TK_UPDATE 320
#define TK_VARIABLE 321 #define TK_VALUES 321
#define TK_VIEW 322 #define TK_VARIABLE 322
#define TK_WAL 323 #define TK_VIEW 323
#define TK_WAL 324
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

1
include/common/ttypes.h
View File

@ -266,6 +266,7 @@ typedef struct {
#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)
#define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)))
#define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP) #define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP)
#define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL)
#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t)))
#define IS_MATHABLE_TYPE(_t) \ #define IS_MATHABLE_TYPE(_t) \

1
include/libs/function/functionMgt.h
View File

@ -120,6 +120,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_WEND, FUNCTION_TYPE_WEND,
FUNCTION_TYPE_WDURATION, FUNCTION_TYPE_WDURATION,
FUNCTION_TYPE_IROWTS, FUNCTION_TYPE_IROWTS,
FUNCTION_TYPE_ISFILLED,
FUNCTION_TYPE_TAGS, FUNCTION_TYPE_TAGS,
// internal function // internal function

10
include/libs/nodes/cmdnodes.h
View File

@ -171,10 +171,10 @@ typedef struct SCreateSubTableClause {
STableOptions* pOptions; STableOptions* pOptions;
} SCreateSubTableClause; } SCreateSubTableClause;
typedef struct SCreateMultiTableStmt { typedef struct SCreateMultiTablesStmt {
ENodeType type; ENodeType type;
SNodeList* pSubTables; SNodeList* pSubTables;
} SCreateMultiTableStmt; } SCreateMultiTablesStmt;
typedef struct SDropTableClause { typedef struct SDropTableClause {
ENodeType type; ENodeType type;
@ -209,14 +209,14 @@ typedef struct SAlterTableStmt {
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t sysinfo; int8_t sysinfo;
} SCreateUserStmt; } SCreateUserStmt;
typedef struct SAlterUserStmt { typedef struct SAlterUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
int8_t alterType; int8_t alterType;
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t enable; int8_t enable;
@ -225,7 +225,7 @@ typedef struct SAlterUserStmt {
typedef struct SDropUserStmt { typedef struct SDropUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
} SDropUserStmt; } SDropUserStmt;
typedef struct SCreateDnodeStmt { typedef struct SCreateDnodeStmt {

4
include/libs/nodes/nodes.h
View File

@ -117,7 +117,7 @@ typedef enum ENodeType {
// Statement nodes are used in parser and planner module. // Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SET_OPERATOR = 100,
QUERY_NODE_SELECT_STMT, QUERY_NODE_SELECT_STMT,
QUERY_NODE_VNODE_MODIF_STMT, QUERY_NODE_VNODE_MODIFY_STMT,
QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_CREATE_DATABASE_STMT,
QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT,
QUERY_NODE_ALTER_DATABASE_STMT, QUERY_NODE_ALTER_DATABASE_STMT,
@ -125,7 +125,7 @@ typedef enum ENodeType {
QUERY_NODE_TRIM_DATABASE_STMT, QUERY_NODE_TRIM_DATABASE_STMT,
QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_TABLE_STMT,
QUERY_NODE_CREATE_SUBTABLE_CLAUSE, QUERY_NODE_CREATE_SUBTABLE_CLAUSE,
QUERY_NODE_CREATE_MULTI_TABLE_STMT, QUERY_NODE_CREATE_MULTI_TABLES_STMT,
QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_CLAUSE,
QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_TABLE_STMT,
QUERY_NODE_DROP_SUPER_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT,

4
include/libs/nodes/querynodes.h
View File

@ -364,7 +364,7 @@ typedef struct SVgDataBlocks {
typedef void (*FFreeTableBlockHash)(SHashObj*); typedef void (*FFreeTableBlockHash)(SHashObj*);
typedef void (*FFreeVgourpBlockArray)(SArray*); typedef void (*FFreeVgourpBlockArray)(SArray*);
typedef struct SVnodeModifOpStmt { typedef struct SVnodeModifyOpStmt {
ENodeType nodeType; ENodeType nodeType;
ENodeType sqlNodeType; ENodeType sqlNodeType;
SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>. SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>.
@ -388,7 +388,7 @@ typedef struct SVnodeModifOpStmt {
FFreeVgourpBlockArray freeArrayFunc; FFreeVgourpBlockArray freeArrayFunc;
bool usingTableProcessing; bool usingTableProcessing;
bool fileProcessing; bool fileProcessing;
} SVnodeModifOpStmt; } SVnodeModifyOpStmt;
typedef struct SExplainOptions { typedef struct SExplainOptions {
ENodeType type; ENodeType type;

4
include/libs/stream/streamState.h
View File

@ -35,6 +35,7 @@ typedef struct STdbState {
TTB* pFillStateDb; // todo refactor TTB* pFillStateDb; // todo refactor
TTB* pSessionStateDb; TTB* pSessionStateDb;
TTB* pParNameDb; TTB* pParNameDb;
TTB* pParTagDb;
TXN* txn; TXN* txn;
} STdbState; } STdbState;
@ -108,6 +109,9 @@ int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur);
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname);
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen);
int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen);
#if 0 #if 0
char* streamStateSessionDump(SStreamState* pState); char* streamStateSessionDump(SStreamState* pState);
#endif #endif

13
packaging/release.sh
View File

@ -3,7 +3,7 @@
# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os # Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
set -e set -e
#set -x # set -x
# release.sh -v [cluster | edge] # release.sh -v [cluster | edge]
# -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...] # -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...]
@ -96,6 +96,8 @@ while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do
esac esac
done done
osType=$(uname)
echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp} httpdBuild=${httpdBuild}" echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp} httpdBuild=${httpdBuild}"
curr_dir=$(pwd) curr_dir=$(pwd)
@ -233,7 +235,12 @@ else
exit 1 exit 1
fi fi
CORES=$(grep -c ^processor /proc/cpuinfo) ostype=`uname`
if [ "${ostype}" == "Darwin" ]; then
CORES=$(sysctl -n hw.ncpu)
else
CORES=$(grep -c ^processor /proc/cpuinfo)
fi
if [[ "$allocator" == "jemalloc" ]]; then if [[ "$allocator" == "jemalloc" ]]; then
# jemalloc need compile first, so disable parallel build # jemalloc need compile first, so disable parallel build
@ -306,7 +313,7 @@ if [ "$osType" != "Darwin" ]; then
${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
else else
# only make client for Darwin
cd ${script_dir}/tools cd ${script_dir}/tools
./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName}
./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName}
fi fi

8
packaging/tools/makeclient.sh
View File

@ -2,7 +2,7 @@
# #
# Generate tar.gz package for linux client in all os system # Generate tar.gz package for linux client in all os system
set -e set -e
#set -x # set -x
curr_dir=$(pwd) curr_dir=$(pwd)
compile_dir=$1 compile_dir=$1
@ -249,9 +249,9 @@ if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :
else else
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || : tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) || :
mv "$(basename ${pkg_name}).tar.gz" .. # mv "$(basename ${pkg_name}).tar.gz" ..
rm -rf ./* rm -rf ${install_dir} ||:
mv ../"$(basename ${pkg_name}).tar.gz" . # mv ../"$(basename ${pkg_name}).tar.gz" .
fi fi
cd ${curr_dir} cd ${curr_dir}

43
packaging/tools/makepkg.sh
View File

@ -3,7 +3,7 @@
# Generate tar.gz package for all os system # Generate tar.gz package for all os system
set -e set -e
#set -x # set -x
curr_dir=$(pwd) curr_dir=$(pwd)
compile_dir=$1 compile_dir=$1
@ -74,14 +74,16 @@ else
tdinsight_caches="" tdinsight_caches=""
cd ${build_dir}/bin/ && \ cd ${build_dir}/bin/ && \
chmod +x TDinsight.sh chmod +x TDinsight.sh
tdinsight_caches=$(./TDinsight.sh --download-only | xargs -i printf "${build_dir}/bin/{} ") ./TDinsight.sh --download-only ||:
# tdinsight_caches=$(./TDinsight.sh --download-only | xargs -I printf "${build_dir}/bin/{} ")
cd $orig_pwd cd $orig_pwd
echo "TDinsight caches: $tdinsight_caches" echo "TDinsight caches: $tdinsight_caches"
taostools_bin_files=" ${build_dir}/bin/taosdump \ taostools_bin_files=" ${build_dir}/bin/taosdump \
${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/taosBenchmark \
${build_dir}/bin/TDinsight.sh \ ${build_dir}/bin/TDinsight.sh \
$tdinsight_caches" ${build_dir}/bin/tdengine-datasource.zip \
${build_dir}/bin/tdengine-datasource.zip.md5sum"
[ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx"
bin_files="${build_dir}/bin/${serverName} \ bin_files="${build_dir}/bin/${serverName} \
@ -96,8 +98,13 @@ else
${script_dir}/taosd-dump-cfg.gdb" ${script_dir}/taosd-dump-cfg.gdb"
fi fi
lib_files="${build_dir}/lib/libtaos.so.${version}" if [ "$osType" == "Darwin" ]; then
wslib_files="${build_dir}/lib/libtaosws.so" lib_files="${build_dir}/lib/libtaos.${version}.dylib"
wslib_files="${build_dir}/lib/libtaosws.dylib"
else
lib_files="${build_dir}/lib/libtaos.so.${version}"
wslib_files="${build_dir}/lib/libtaosws.so"
fi
header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h"
wsheader_files="${build_dir}/include/taosws.h" wsheader_files="${build_dir}/include/taosws.h"
@ -226,7 +233,12 @@ if [ "$verMode" == "cloud" ]; then
fi fi
cd ${install_dir} cd ${install_dir}
tar -zcv -f ${tarName} * --remove-files || : if [ "$osType" != "Darwin" ]; then
tar -zcv -f ${tarName} * --remove-files || :
else
tar -zcv -f ${tarName} * || :
fi
exitcode=$? exitcode=$?
if [ "$exitcode" != "0" ]; then if [ "$exitcode" != "0" ]; then
echo "tar ${tarName} error !!!" echo "tar ${tarName} error !!!"
@ -288,7 +300,7 @@ if [[ $dbName == "taos" ]]; then
if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then
if [ -d "${web_dir}/admin" ] ; then if [ -d "${web_dir}/admin" ] ; then
mkdir -p ${install_dir}/share/ mkdir -p ${install_dir}/share/
cp ${web_dir}/admin ${install_dir}/share/ -r cp -Rfap ${web_dir}/admin ${install_dir}/share/
cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png
else else
echo "directory not found for enterprise release: ${web_dir}/admin" echo "directory not found for enterprise release: ${web_dir}/admin"
@ -362,7 +374,15 @@ if [ "$pagMode" == "lite" ]; then
pkg_name=${pkg_name}-Lite pkg_name=${pkg_name}-Lite
fi fi
tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" --remove-files || :
if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" --remove-files || :
else
tar -zcv -f "$(basename ${pkg_name}).tar.gz" "$(basename ${install_dir})" || :
rm -rf ${install_dir} ||:
([ -d build-taoskeeper ] && rm -rf build-taoskeeper ) ||:
fi
exitcode=$? exitcode=$?
if [ "$exitcode" != "0" ]; then if [ "$exitcode" != "0" ]; then
echo "tar ${pkg_name}.tar.gz error !!!" echo "tar ${pkg_name}.tar.gz error !!!"
@ -371,7 +391,12 @@ fi
if [ -n "${taostools_bin_files}" ]; then if [ -n "${taostools_bin_files}" ]; then
wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh" wget https://github.com/taosdata/grafanaplugin/releases/latest/download/TDinsight.sh -O ${taostools_install_dir}/bin/TDinsight.sh && echo "TDinsight.sh downloaded!"|| echo "failed to download TDinsight.sh"
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || : if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" --remove-files || :
else
tar -zcv -f "$(basename ${taostools_pkg_name}).tar.gz" "$(basename ${taostools_install_dir})" || :
rm -rf ${taostools_install_dir} ||:
fi
exitcode=$? exitcode=$?
if [ "$exitcode" != "0" ]; then if [ "$exitcode" != "0" ]; then
echo "tar ${taostools_pkg_name}.tar.gz error !!!" echo "tar ${taostools_pkg_name}.tar.gz error !!!"

8
source/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h
View File

@ -260,6 +260,14 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_insertLinesImp(JN
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertImp(JNIEnv *, jobject, jobjectArray, JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsertImp(JNIEnv *, jobject, jobjectArray,
jlong, jint, jint); jlong, jint, jint);
/**
* Class: com_taosdata_jdbc_TSDBJNIConnector
* Method: getTableVgID
* Signature: (Ljava/lang/String;Ljava/lang/String)Lcom/taosdata/jdbc/VGroupIDResp
*/
JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTableVgID(JNIEnv *, jobject, jlong, jstring,
jstring, jobject);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

24
source/client/src/clientEnv.c
View File

@ -77,19 +77,18 @@ static void deregisterRequest(SRequestObj *pRequest) {
pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst);
tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
"us, exec:%" PRId64 "us, stmtType:%d", "us, exec:%" PRId64 "us, stmtType:%d",
duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd,
pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd, pRequest->stmtType);
pRequest->stmtType);
if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType) {
// tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
// "us, exec:%" PRId64 "us", // "us, exec:%" PRId64 "us",
// duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart,
// pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd -
// pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd);
// atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); // atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration);
} else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) {
// tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64
// "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64,
@ -475,6 +474,9 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
case TSDB_OPTION_TIMEZONE: case TSDB_OPTION_TIMEZONE:
pItem = cfgGetItem(pCfg, "timezone"); pItem = cfgGetItem(pCfg, "timezone");
break; break;
case TSDB_OPTION_USE_ADAPTER:
pItem = cfgGetItem(pCfg, "useAdapter");
break;
default: default:
break; break;
} }

8
source/client/src/clientImpl.c
View File

@ -734,7 +734,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
} }
int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) {
SArray* pArray = NULL; SArray* pArray = NULL;
SSubmitRsp2* pRsp = (SSubmitRsp2*)res; SSubmitRsp2* pRsp = (SSubmitRsp2*)res;
if (NULL == pRsp->aCreateTbRsp) { if (NULL == pRsp->aCreateTbRsp) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -849,7 +849,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
} }
static bool incompletaFileParsing(SNode* pStmt) { static bool incompletaFileParsing(SNode* pStmt) {
return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing; return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing;
} }
// todo refacto the error code mgmt // todo refacto the error code mgmt
@ -928,7 +928,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
if (pQuery->pRoot && !pRequest->inRetry) { if (pQuery->pRoot && !pRequest->inRetry) {
STscObj* pTscObj = pRequest->pTscObj; STscObj* pTscObj = pRequest->pTscObj;
SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary;
if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type) { if (QUERY_NODE_VNODE_MODIFY_STMT == pQuery->pRoot->type) {
atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertsReq, 1); atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertsReq, 1);
} else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) {
atomic_add_fetch_64((int64_t*)&pActivity->numOfQueryReq, 1); atomic_add_fetch_64((int64_t*)&pActivity->numOfQueryReq, 1);
@ -1033,7 +1033,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
} }
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
SArray* pNodeList = NULL; SArray* pNodeList = NULL;
if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pQuery->pRoot)) { if (QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pQuery->pRoot)) {
buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta);
} }

65
source/client/src/clientJniConnector.c
View File

@ -488,7 +488,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
numOfFields); numOfFields);
return JNI_FETCH_END; return JNI_FETCH_END;
} else { } else {
jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code, taos_errstr(result)); jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code,
taos_errstr(result));
return JNI_RESULT_SET_NULL; return JNI_RESULT_SET_NULL;
} }
} }
@ -583,7 +584,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res);
return JNI_FETCH_END; return JNI_FETCH_END;
} else { } else {
jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code,
taos_errstr(tres));
return JNI_RESULT_SET_NULL; return JNI_RESULT_SET_NULL;
} }
} }
@ -1028,3 +1030,62 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInsert
} }
return (jlong)tres; return (jlong)tres;
} }
// TABLE_VG_ID_FID_CACHE cache resp object for getTableVgID
typedef struct TABLE_VG_ID_FIELD_CACHE {
int cached;
jclass clazz;
jfieldID codeField;
jfieldID vgIDField;
} TABLE_VG_ID_FIELD_CACHE;
TABLE_VG_ID_FIELD_CACHE tableVgIdFieldCache;
void cacheTableVgIDField(JNIEnv *env, jobject jobj) {
if (tableVgIdFieldCache.cached) {
return;
}
tableVgIdFieldCache.clazz = (*env)->GetObjectClass(env, jobj);
tableVgIdFieldCache.codeField = (*env)->GetFieldID(env, tableVgIdFieldCache.clazz, "code", "I");
tableVgIdFieldCache.vgIDField = (*env)->GetFieldID(env, tableVgIdFieldCache.clazz, "vgID", "I");
tableVgIdFieldCache.cached = 1;
}
JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getTableVgID(JNIEnv *env, jobject jobj, jlong conn,
jstring jdb, jstring jtable,
jobject resp) {
if (!tableVgIdFieldCache.cached) {
cacheTableVgIDField(env, resp);
}
TAOS *taos = (TAOS *)conn;
if (taos == NULL) {
jniError("jobj:%p, connection already closed", jobj);
(*env)->SetIntField(env, resp, tableVgIdFieldCache.codeField, JNI_CONNECTION_NULL);
return resp;
}
const char *db = NULL;
const char *table = NULL;
int vgID = 0;
if (jdb != NULL) {
db = (*env)->GetStringUTFChars(env, jdb, NULL);
}
if (jtable != NULL) {
table = (*env)->GetStringUTFChars(env, jtable, NULL);
}
int code = taos_get_table_vgId(taos, db, table, &vgID);
if (db != NULL) {
(*env)->ReleaseStringUTFChars(env, jdb, db);
}
if (table != NULL) {
(*env)->ReleaseStringUTFChars(env, jtable, table);
}
(*env)->SetIntField(env, resp, tableVgIdFieldCache.codeField, code);
(*env)->SetIntField(env, resp, tableVgIdFieldCache.vgIDField, vgID);
return resp;
}

22
source/client/src/clientRawBlockWrite.c
View File

@ -1201,7 +1201,8 @@ end:
return code; return code;
} }
int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields,
int numFields) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
@ -1251,7 +1252,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
goto end; goto end;
} }
pQuery = smlInitHandle(); pQuery = smlInitHandle();
if(pQuery == NULL){ if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
@ -1273,7 +1274,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
launchQueryImpl(pRequest, pQuery, true, NULL); launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code; code = pRequest->code;
end: end:
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
destroyRequest(pRequest); destroyRequest(pRequest);
@ -1331,7 +1332,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
goto end; goto end;
} }
pQuery = smlInitHandle(); pQuery = smlInitHandle();
if(pQuery == NULL){ if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
@ -1408,7 +1409,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); pQuery = smlInitHandle();
if(pQuery == NULL){ if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
@ -1454,7 +1455,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) {
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) { if (hData == NULL) {
taosHashPut(pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg));
} }
code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, NULL, 0); code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, NULL, 0);
@ -1483,7 +1484,6 @@ end:
return code; return code;
} }
static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SHashObj* pVgHash = NULL; SHashObj* pVgHash = NULL;
@ -1531,7 +1531,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp);
pQuery = smlInitHandle(); pQuery = smlInitHandle();
if(pQuery == NULL){ if (pQuery == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto end; goto end;
} }
@ -1564,7 +1564,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j);
int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j);
SDecoder decoderTmp = {0}; SDecoder decoderTmp = {0};
tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); tDecoderInit(&decoderTmp, *dataTmp, *lenTmp);
memset(&pCreateReq, 0, sizeof(SVCreateTbReq)); memset(&pCreateReq, 0, sizeof(SVCreateTbReq));
if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) {
@ -1601,7 +1601,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId));
if (hData == NULL) { if (hData == NULL) {
taosHashPut(pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg));
} }
code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, &pCreateReq, NULL, 0); code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, &pCreateReq, NULL, 0);
@ -1614,7 +1614,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen)
launchQueryImpl(pRequest, pQuery, true, NULL); launchQueryImpl(pRequest, pQuery, true, NULL);
code = pRequest->code; code = pRequest->code;
end: end:
tDeleteSTaosxRsp(&rspObj.rsp); tDeleteSTaosxRsp(&rspObj.rsp);
tDecoderClear(&decoder); tDecoderClear(&decoder);
qDestroyQuery(pQuery); qDestroyQuery(pQuery);

457
source/client/src/clientSml.c
View File

@ -24,15 +24,15 @@ int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn){ void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
NodeList *tmp = list; NodeList *tmp = list;
while(tmp){ while (tmp) {
if(fn == NULL){ if (fn == NULL) {
if(tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
return tmp->data.value; return tmp->data.value;
} }
}else{ } else {
if(tmp->data.used && fn(tmp->data.key, key) == 0) { if (tmp->data.used && fn(tmp->data.key, key) == 0) {
return tmp->data.value; return tmp->data.value;
} }
} }
@ -42,30 +42,30 @@ void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn
return NULL; return NULL;
} }
int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn){ int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
NodeList *tmp = *list; NodeList *tmp = *list;
while (tmp){ while (tmp) {
if(!tmp->data.used) break; if (!tmp->data.used) break;
if(fn == NULL){ if (fn == NULL) {
if(tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
return -1; return -1;
} }
}else{ } else {
if(tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
return -1; return -1;
} }
} }
tmp = tmp->next; tmp = tmp->next;
} }
if(tmp){ if (tmp) {
tmp->data.key = key; tmp->data.key = key;
tmp->data.keyLen = len; tmp->data.keyLen = len;
tmp->data.value = value; tmp->data.value = value;
tmp->data.used = true; tmp->data.used = true;
}else{ } else {
NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
if(newNode == NULL){ if (newNode == NULL) {
return -1; return -1;
} }
newNode->data.key = key; newNode->data.key = key;
@ -78,11 +78,13 @@ int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equ
return 0; return 0;
} }
int nodeListSize(NodeList* list){ int nodeListSize(NodeList *list) {
int cnt = 0; int cnt = 0;
while(list){ while (list) {
if(list->data.used) cnt++; if (list->data.used)
else break; cnt++;
else
break;
list = list->next; list = list->next;
} }
return cnt; return cnt;
@ -106,7 +108,6 @@ int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *m
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) {
char *endPtr = NULL; char *endPtr = NULL;
int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10);
@ -114,9 +115,9 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
return -1; return -1;
} }
if(unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)){ if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) {
int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS];
if(unit > INT64_MAX / tsInt64){ if (unit > INT64_MAX / tsInt64) {
return -1; return -1;
} }
tsInt64 *= unit; tsInt64 *= unit;
@ -136,7 +137,7 @@ int8_t smlGetTsTypeByLen(int32_t len) {
} }
} }
SSmlTableInfo *smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen) { SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
if (!tag) { if (!tag) {
return NULL; return NULL;
@ -151,23 +152,23 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char* measure, int32_t measu
goto cleanup; goto cleanup;
} }
// tag->tags = taosArrayInit(16, sizeof(SSmlKv)); // tag->tags = taosArrayInit(16, sizeof(SSmlKv));
// if (tag->tags == NULL) { // if (tag->tags == NULL) {
// uError("SML:smlBuildTableInfo failed to allocate memory"); // uError("SML:smlBuildTableInfo failed to allocate memory");
// goto cleanup; // goto cleanup;
// } // }
return tag; return tag;
cleanup: cleanup:
taosMemoryFree(tag); taosMemoryFree(tag);
return NULL; return NULL;
} }
static int32_t smlParseTableName(SArray *tags, char *childTableName) { static int32_t smlParseTableName(SArray *tags, char *childTableName) {
size_t childTableNameLen = strlen(tsSmlChildTableName); size_t childTableNameLen = strlen(tsSmlChildTableName);
if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS;
for(int i = 0; i < taosArrayGetSize(tags); i++){ for (int i = 0; i < taosArrayGetSize(tags); i++) {
SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
// handle child table name // handle child table name
if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
@ -180,13 +181,12 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlSetCTableName(SSmlTableInfo *oneTable){ int32_t smlSetCTableName(SSmlTableInfo *oneTable) {
smlParseTableName(oneTable->tags, oneTable->childTableName); smlParseTableName(oneTable->tags, oneTable->childTableName);
if (strlen(oneTable->childTableName) == 0) { if (strlen(oneTable->childTableName) == 0) {
SArray* dst = taosArrayDup(oneTable->tags, NULL); SArray *dst = taosArrayDup(oneTable->tags, NULL);
RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName, 0};
oneTable->childTableName, 0};
buildChildTableName(&rName); buildChildTableName(&rName);
taosArrayDestroy(dst); taosArrayDestroy(dst);
@ -203,7 +203,7 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
return NULL; return NULL;
} }
if(unlikely(!isDataFormat)){ if (unlikely(!isDataFormat)) {
meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
if (meta->tagHash == NULL) { if (meta->tagHash == NULL) {
uError("SML:smlBuildSTableMeta failed to allocate memory"); uError("SML:smlBuildSTableMeta failed to allocate memory");
@ -230,116 +230,117 @@ SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) {
} }
return meta; return meta;
cleanup: cleanup:
taosMemoryFree(meta); taosMemoryFree(meta);
return NULL; return NULL;
} }
//uint16_t smlCalTypeSum(char* endptr, int32_t left){ // uint16_t smlCalTypeSum(char* endptr, int32_t left){
// uint16_t sum = 0; // uint16_t sum = 0;
// for(int i = 0; i < left; i++){ // for(int i = 0; i < left; i++){
// sum += endptr[i]; // sum += endptr[i];
// } // }
// return sum; // return sum;
//} // }
#define RETURN_FALSE \ #define RETURN_FALSE \
smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
return false; return false;
#define SET_DOUBLE kvVal->type = TSDB_DATA_TYPE_DOUBLE;\ #define SET_DOUBLE \
kvVal->d = result; kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
kvVal->d = result;
#define SET_FLOAT \ #define SET_FLOAT \
if (!IS_VALID_FLOAT(result)) {\ if (!IS_VALID_FLOAT(result)) { \
smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);\ smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_FLOAT;\ kvVal->type = TSDB_DATA_TYPE_FLOAT; \
kvVal->f = (float)result; kvVal->f = (float)result;
#define SET_BIGINT \ #define SET_BIGINT \
if (smlDoubleToInt64OverFlow(result)) {\ if (smlDoubleToInt64OverFlow(result)) { \
errno = 0;\ errno = 0; \
int64_t tmp = taosStr2Int64(pVal, &endptr, 10);\ int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \
if (errno == ERANGE) {\ if (errno == ERANGE) { \
smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);\ smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_BIGINT;\ kvVal->type = TSDB_DATA_TYPE_BIGINT; \
kvVal->i = tmp;\ kvVal->i = tmp; \
return true;\ return true; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_BIGINT;\ kvVal->type = TSDB_DATA_TYPE_BIGINT; \
kvVal->i = (int64_t)result; kvVal->i = (int64_t)result;
#define SET_INT \ #define SET_INT \
if (!IS_VALID_INT(result)) {\ if (!IS_VALID_INT(result)) { \
smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);\ smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_INT;\ kvVal->type = TSDB_DATA_TYPE_INT; \
kvVal->i = result; kvVal->i = result;
#define SET_SMALL_INT \ #define SET_SMALL_INT \
if (!IS_VALID_SMALLINT(result)) {\ if (!IS_VALID_SMALLINT(result)) { \
smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);\ smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_SMALLINT;\ kvVal->type = TSDB_DATA_TYPE_SMALLINT; \
kvVal->i = result; kvVal->i = result;
#define SET_UBIGINT \ #define SET_UBIGINT \
if (result >= (double)UINT64_MAX || result < 0) {\ if (result >= (double)UINT64_MAX || result < 0) { \
errno = 0;\ errno = 0; \
uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);\ uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \
if (errno == ERANGE || result < 0) {\ if (errno == ERANGE || result < 0) { \
smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);\ smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
kvVal->u = tmp;\ kvVal->u = tmp; \
return true;\ return true; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ kvVal->type = TSDB_DATA_TYPE_UBIGINT; \
kvVal->u = result; kvVal->u = result;
#define SET_UINT \ #define SET_UINT \
if (!IS_VALID_UINT(result)) {\ if (!IS_VALID_UINT(result)) { \
smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);\ smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_UINT;\ kvVal->type = TSDB_DATA_TYPE_UINT; \
kvVal->u = result; kvVal->u = result;
#define SET_USMALL_INT \ #define SET_USMALL_INT \
if (!IS_VALID_USMALLINT(result)) {\ if (!IS_VALID_USMALLINT(result)) { \
smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);\ smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_USMALLINT;\ kvVal->type = TSDB_DATA_TYPE_USMALLINT; \
kvVal->u = result; kvVal->u = result;
#define SET_TINYINT \ #define SET_TINYINT \
if (!IS_VALID_TINYINT(result)) { \ if (!IS_VALID_TINYINT(result)) { \
smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);\ smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_TINYINT;\ kvVal->type = TSDB_DATA_TYPE_TINYINT; \
kvVal->i = result; kvVal->i = result;
#define SET_UTINYINT \ #define SET_UTINYINT \
if (!IS_VALID_UTINYINT(result)) {\ if (!IS_VALID_UTINYINT(result)) { \
smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);\ smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
return false;\ return false; \
}\ } \
kvVal->type = TSDB_DATA_TYPE_UTINYINT;\ kvVal->type = TSDB_DATA_TYPE_UTINYINT; \
kvVal->u = result; kvVal->u = result;
bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
const char *pVal = kvVal->value; const char *pVal = kvVal->value;
int32_t len = kvVal->length; int32_t len = kvVal->length;
char * endptr = NULL; char *endptr = NULL;
double result = taosStr2Double(pVal, &endptr); double result = taosStr2Double(pVal, &endptr);
if (pVal == endptr) { if (pVal == endptr) {
RETURN_FALSE RETURN_FALSE
@ -349,59 +350,59 @@ bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
if (left == 0) { if (left == 0) {
SET_DOUBLE SET_DOUBLE
} else if (left == 3) { } else if (left == 3) {
if(endptr[0] == 'f' || endptr[0] == 'F'){ if (endptr[0] == 'f' || endptr[0] == 'F') {
if(endptr[1] == '6' && endptr[2] == '4'){ if (endptr[1] == '6' && endptr[2] == '4') {
SET_DOUBLE SET_DOUBLE
}else if(endptr[1] == '3' && endptr[2] == '2'){ } else if (endptr[1] == '3' && endptr[2] == '2') {
SET_FLOAT SET_FLOAT
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
}else if(endptr[0] == 'i' || endptr[0] == 'I'){ } else if (endptr[0] == 'i' || endptr[0] == 'I') {
if(endptr[1] == '6' && endptr[2] == '4'){ if (endptr[1] == '6' && endptr[2] == '4') {
SET_BIGINT SET_BIGINT
}else if(endptr[1] == '3' && endptr[2] == '2'){ } else if (endptr[1] == '3' && endptr[2] == '2') {
SET_INT SET_INT
}else if(endptr[1] == '1' && endptr[2] == '6'){ } else if (endptr[1] == '1' && endptr[2] == '6') {
SET_SMALL_INT SET_SMALL_INT
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
}else if(endptr[0] == 'u' || endptr[0] == 'U'){ } else if (endptr[0] == 'u' || endptr[0] == 'U') {
if(endptr[1] == '6' && endptr[2] == '4'){ if (endptr[1] == '6' && endptr[2] == '4') {
SET_UBIGINT SET_UBIGINT
}else if(endptr[1] == '3' && endptr[2] == '2'){ } else if (endptr[1] == '3' && endptr[2] == '2') {
SET_UINT SET_UINT
}else if(endptr[1] == '1' && endptr[2] == '6'){ } else if (endptr[1] == '1' && endptr[2] == '6') {
SET_USMALL_INT SET_USMALL_INT
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
} else if(left == 2){ } else if (left == 2) {
if(endptr[0] == 'i' || endptr[0] == 'I'){ if (endptr[0] == 'i' || endptr[0] == 'I') {
if(endptr[1] == '8') { if (endptr[1] == '8') {
SET_TINYINT SET_TINYINT
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
}else if(endptr[0] == 'u' || endptr[0] == 'U') { } else if (endptr[0] == 'u' || endptr[0] == 'U') {
if (endptr[1] == '8') { if (endptr[1] == '8') {
SET_UTINYINT SET_UTINYINT
} else { } else {
RETURN_FALSE RETURN_FALSE
} }
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
} else if(left == 1){ } else if (left == 1) {
if(endptr[0] == 'i' || endptr[0] == 'I'){ if (endptr[0] == 'i' || endptr[0] == 'I') {
SET_BIGINT SET_BIGINT
}else if(endptr[0] == 'u' || endptr[0] == 'U') { } else if (endptr[0] == 'u' || endptr[0] == 'U') {
SET_UBIGINT SET_UBIGINT
}else{ } else {
RETURN_FALSE RETURN_FALSE
} }
} else { } else {
@ -508,7 +509,7 @@ bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) {
return true; return true;
} }
STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen){ STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) {
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}};
@ -526,7 +527,6 @@ STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen
return pTableMeta; return pTableMeta;
} }
static int64_t smlGenId() { static int64_t smlGenId() {
static volatile int64_t linesSmlHandleId = 0; static volatile int64_t linesSmlHandleId = 0;
@ -736,14 +736,14 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
code = pRequest->code; code = pRequest->code;
taosMemoryFree(pCmdMsg.pMsg); taosMemoryFree(pCmdMsg.pMsg);
end: end:
destroyRequest(pRequest); destroyRequest(pRequest);
tFreeSMCreateStbReq(&pReq); tFreeSMCreateStbReq(&pReq);
return code; return code;
} }
static int32_t smlModifyDBSchemas(SSmlHandle *info) { static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if(info->dataFormat && !info->needModifySchema){ if (info->dataFormat && !info->needModifySchema) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = 0; int32_t code = 0;
@ -764,8 +764,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value; SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value;
bool needCheckMeta = false; // for multi thread bool needCheckMeta = false; // for multi thread
size_t superTableLen = (size_t)tmp->data.keyLen; size_t superTableLen = (size_t)tmp->data.keyLen;
const void *superTable = tmp->data.key; const void *superTable = tmp->data.key;
memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memset(pName.tname, 0, TSDB_TABLE_NAME_LEN);
memcpy(pName.tname, superTable, superTableLen); memcpy(pName.tname, superTable, superTableLen);
@ -918,14 +918,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
return 0; return 0;
end: end:
taosHashCleanup(hashTmp); taosHashCleanup(hashTmp);
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
// catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1);
return code; return code;
} }
/* /*
static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
for(int i = 0; i < taosArrayGetSize(tags); i++) { for(int i = 0; i < taosArrayGetSize(tags); i++) {
@ -955,11 +954,11 @@ static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
} }
*/ */
static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols){ static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES);
if(ret == 0){ if (ret == 0) {
taosArrayPush(metaArray, kv); taosArrayPush(metaArray, kv);
} }
} }
@ -981,7 +980,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen);
if (index) { if (index) {
SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index);
if (isTag){ if (isTag) {
if (kv->length > value->length) { if (kv->length > value->length) {
value->length = kv->length; value->length = kv->length;
} }
@ -999,8 +998,8 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
size_t tmp = taosArrayGetSize(metaArray); size_t tmp = taosArrayGetSize(metaArray);
ASSERT(tmp <= INT16_MAX); ASSERT(tmp <= INT16_MAX);
int16_t size = tmp; int16_t size = tmp;
int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES);
if(ret == 0){ if (ret == 0) {
taosArrayPush(metaArray, kv); taosArrayPush(metaArray, kv);
} }
} }
@ -1026,12 +1025,12 @@ void smlDestroyInfo(SSmlHandle *info) {
qDestroyQuery(info->pQuery); qDestroyQuery(info->pQuery);
// destroy info->childTables // destroy info->childTables
NodeList* tmp = info->childTables; NodeList *tmp = info->childTables;
while (tmp) { while (tmp) {
if(tmp->data.used) { if (tmp->data.used) {
smlDestroyTableInfo((SSmlTableInfo*)tmp->data.value); smlDestroyTableInfo((SSmlTableInfo *)tmp->data.value);
} }
NodeList* t = tmp->next; NodeList *t = tmp->next;
taosMemoryFree(tmp); taosMemoryFree(tmp);
tmp = t; tmp = t;
} }
@ -1039,10 +1038,10 @@ void smlDestroyInfo(SSmlHandle *info) {
// destroy info->superTables // destroy info->superTables
tmp = info->superTables; tmp = info->superTables;
while (tmp) { while (tmp) {
if(tmp->data.used) { if (tmp->data.used) {
smlDestroySTableMeta((SSmlSTableMeta*)tmp->data.value); smlDestroySTableMeta((SSmlSTableMeta *)tmp->data.value);
} }
NodeList* t = tmp->next; NodeList *t = tmp->next;
taosMemoryFree(tmp); taosMemoryFree(tmp);
tmp = t; tmp = t;
} }
@ -1053,8 +1052,8 @@ void smlDestroyInfo(SSmlHandle *info) {
taosArrayDestroy(info->preLineTagKV); taosArrayDestroy(info->preLineTagKV);
taosArrayDestroy(info->preLineColKV); taosArrayDestroy(info->preLineColKV);
if(!info->dataFormat){ if (!info->dataFormat) {
for(int i = 0; i < info->lineNum; i++){ for (int i = 0; i < info->lineNum; i++) {
taosArrayDestroy(info->lines[i].colArray); taosArrayDestroy(info->lines[i].colArray);
} }
taosMemoryFree(info->lines); taosMemoryFree(info->lines);
@ -1069,7 +1068,7 @@ SSmlHandle *smlBuildSmlInfo(TAOS *taos) {
if (NULL == info) { if (NULL == info) {
return NULL; return NULL;
} }
if(taos != NULL){ if (taos != NULL) {
info->taos = acquireTscObj(*(int64_t *)taos); info->taos = acquireTscObj(*(int64_t *)taos);
code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -1114,20 +1113,20 @@ static int32_t smlPushCols(SArray *colsArray, SArray *cols) {
} }
static int32_t smlParseLineBottom(SSmlHandle *info) { static int32_t smlParseLineBottom(SSmlHandle *info) {
if(info->dataFormat) return TSDB_CODE_SUCCESS; if (info->dataFormat) return TSDB_CODE_SUCCESS;
for(int32_t i = 0; i < info->lineNum; i ++){ for (int32_t i = 0; i < info->lineNum; i++) {
SSmlLineInfo* elements = info->lines + i; SSmlLineInfo *elements = info->lines + i;
SSmlTableInfo *tinfo = NULL; SSmlTableInfo *tinfo = NULL;
if(info->protocol == TSDB_SML_LINE_PROTOCOL){ if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL);
}else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
}else{ } else {
tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet);
} }
if(tinfo == NULL){ if (tinfo == NULL) {
uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i);
smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
@ -1148,7 +1147,8 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
return ret; return ret;
} }
SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); SSmlSTableMeta *tableMeta =
(SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL);
if (tableMeta) { // update meta if (tableMeta) { // update meta
ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf);
if (ret == TSDB_CODE_SUCCESS) { if (ret == TSDB_CODE_SUCCESS) {
@ -1159,11 +1159,11 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
return ret; return ret;
} }
} else { } else {
// ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
// if (ret != TSDB_CODE_SUCCESS) { // if (ret != TSDB_CODE_SUCCESS) {
// uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
// return ret; // return ret;
// } // }
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags);
@ -1175,11 +1175,10 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t smlInsertData(SSmlHandle *info) { static int32_t smlInsertData(SSmlHandle *info) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
NodeList* tmp = info->childTables; NodeList *tmp = info->childTables;
while (tmp) { while (tmp) {
SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value;
@ -1209,9 +1208,9 @@ static int32_t smlInsertData(SSmlHandle *info) {
pMeta->tableMeta->vgId = vg.vgId; pMeta->tableMeta->vgId = vg.vgId;
pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid
code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, pMeta->tableMeta,
pMeta->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl,
info->ttl, info->msgBuf.buf, info->msgBuf.len); info->msgBuf.buf, info->msgBuf.len);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlBindData failed", info->id); uError("SML:0x%" PRIx64 " smlBindData failed", info->id);
return code; return code;
@ -1234,24 +1233,25 @@ static int32_t smlInsertData(SSmlHandle *info) {
} }
static void smlPrintStatisticInfo(SSmlHandle *info) { static void smlPrintStatisticInfo(SSmlHandle *info) {
uError("SML:0x%" PRIx64 uError(
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ "SML:0x%" PRIx64
" smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
"", "",
info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables,
info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables,
info->cost.schemaTime - info->cost.parseTime, info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime,
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime,
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); info->cost.endTime - info->cost.parseTime);
} }
int32_t smlClearForRerun(SSmlHandle *info){ int32_t smlClearForRerun(SSmlHandle *info) {
info->reRun = false; info->reRun = false;
// clear info->childTables // clear info->childTables
NodeList* pList = info->childTables; NodeList *pList = info->childTables;
while (pList) { while (pList) {
if(pList->data.used) { if (pList->data.used) {
smlDestroyTableInfo((SSmlTableInfo*)pList->data.value); smlDestroyTableInfo((SSmlTableInfo *)pList->data.value);
pList->data.used = false; pList->data.used = false;
} }
pList = pList->next; pList = pList->next;
@ -1260,24 +1260,24 @@ int32_t smlClearForRerun(SSmlHandle *info){
// clear info->superTables // clear info->superTables
pList = info->superTables; pList = info->superTables;
while (pList) { while (pList) {
if(pList->data.used) { if (pList->data.used) {
smlDestroySTableMeta((SSmlSTableMeta*)pList->data.value); smlDestroySTableMeta((SSmlSTableMeta *)pList->data.value);
pList->data.used = false; pList->data.used = false;
} }
pList = pList->next; pList = pList->next;
} }
if(unlikely(info->lines != NULL)){ if (unlikely(info->lines != NULL)) {
uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); uError("SML:0x%" PRIx64 " info->lines != NULL", info->id);
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
info->lines = (SSmlLineInfo*)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo));
memset(&info->preLine, 0, sizeof(SSmlLineInfo)); memset(&info->preLine, 0, sizeof(SSmlLineInfo));
info->currSTableMeta = NULL; info->currSTableMeta = NULL;
info->currTableDataCtx = NULL; info->currTableDataCtx = NULL;
SVnodeModifOpStmt* stmt= (SVnodeModifOpStmt*)(info->pQuery->pRoot); SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot);
stmt->freeHashFunc(stmt->pTableBlockHashObj); stmt->freeHashFunc(stmt->pTableBlockHashObj);
stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1298,7 +1298,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
return code; return code;
} }
char *oldRaw = rawLine; char *oldRaw = rawLine;
int32_t i = 0; int32_t i = 0;
while (i < numLines) { while (i < numLines) {
char *tmp = NULL; char *tmp = NULL;
@ -1319,20 +1319,21 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
} }
} }
uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : tmp)); uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len,
(info->isRawLine ? "rawdata" : tmp));
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
if(info->dataFormat){ if (info->dataFormat) {
SSmlLineInfo element = {0}; SSmlLineInfo element = {0};
code = smlParseInfluxString(info, tmp, tmp + len, &element); code = smlParseInfluxString(info, tmp, tmp + len, &element);
}else{ } else {
code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i);
} }
} else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) {
if(info->dataFormat) { if (info->dataFormat) {
SSmlLineInfo element = {0}; SSmlLineInfo element = {0};
code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element);
}else{ } else {
code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i);
} }
@ -1343,11 +1344,11 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
return code; return code;
} }
if(info->reRun){ if (info->reRun) {
i = 0; i = 0;
rawLine = oldRaw; rawLine = oldRaw;
code = smlClearForRerun(info); code = smlClearForRerun(info);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
continue; continue;
@ -1401,8 +1402,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
return code; return code;
} }
TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines,
int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { int protocol, int precision, int32_t ttl, int64_t reqid) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (NULL == taos) { if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
@ -1423,7 +1424,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
} }
info->pRequest = request; info->pRequest = request;
info->isRawLine = rawLine != NULL; info->isRawLine = rawLine != NULL;
info->ttl = ttl; info->ttl = ttl;
info->precision = precision; info->precision = precision;
info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol;
info->msgBuf.buf = info->pRequest->msgBuf; info->msgBuf.buf = info->pRequest->msgBuf;
@ -1462,7 +1463,7 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
request->code = code; request->code = code;
info->cost.endTime = taosGetTimestampUs(); info->cost.endTime = taosGetTimestampUs();
info->cost.code = code; info->cost.code = code;
// smlPrintStatisticInfo(info); // smlPrintStatisticInfo(info);
end: end:
smlDestroyInfo(info); smlDestroyInfo(info);
@ -1497,12 +1498,15 @@ TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int pr
return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0);
} }
TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl) { TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
int32_t ttl) {
return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0); return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0);
} }
TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid) { TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision,
return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); int64_t reqid) {
return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL,
reqid);
} }
TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
@ -1522,13 +1526,18 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int
return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid);
} }
TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); int precision, int64_t reqid) {
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
TSDB_DEFAULT_TABLE_TTL, reqid);
} }
TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl) { TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
int precision, int32_t ttl) {
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0); return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0);
} }
TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision) { TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol,
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); int precision) {
return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision,
TSDB_DEFAULT_TABLE_TTL, 0);
} }

22
source/client/src/clientTmq.c
View File

@ -438,6 +438,7 @@ int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) {
taosMemoryFree(pParam->pOffset); taosMemoryFree(pParam->pOffset);
taosMemoryFree(pBuf->pData); taosMemoryFree(pBuf->pData);
taosMemoryFree(pBuf->pEpSet);
/*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId, /*tscDebug("receive offset commit cb of %s on vgId:%d, offset is %" PRId64, pParam->pOffset->subKey, pParam->->vgId,
* pOffset->version);*/ * pOffset->version);*/
@ -529,7 +530,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT
int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) {
char* topic; char* topic;
int32_t vgId; int32_t vgId;
ASSERT(msg != NULL);
if (TD_RES_TMQ(msg)) { if (TD_RES_TMQ(msg)) {
SMqRspObj* pRspObj = (SMqRspObj*)msg; SMqRspObj* pRspObj = (SMqRspObj*)msg;
topic = pRspObj->topic; topic = pRspObj->topic;
@ -724,7 +724,10 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) {
} }
int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
if (pMsg && pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg) {
taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
}
return 0; return 0;
} }
@ -805,8 +808,6 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId, tmqMgmt.timer, &tmq->commitTimer); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, pRefId, tmqMgmt.timer, &tmq->commitTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
} else {
ASSERT(0);
} }
taosFreeQitem(pTaskType); taosFreeQitem(pTaskType);
} }
@ -869,6 +870,8 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code; pParam->rspErr = code;
taosMemoryFree(pMsg->pEpSet);
tsem_post(&pParam->rspSem); tsem_post(&pParam->rspSem);
return 0; return 0;
} }
@ -947,10 +950,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user; const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user;
const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass;
ASSERT(user);
ASSERT(pass);
ASSERT(conf->groupId[0]);
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
pTmq->mqueue = taosOpenQueue(); pTmq->mqueue = taosOpenQueue();
pTmq->qall = taosAllocateQall(); pTmq->qall = taosAllocateQall();
@ -1166,6 +1165,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
if (code != 0) { if (code != 0) {
tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr()); tscWarn("msg discard from vgId:%d, epoch %d, since %s", vgId, epoch, terrstr());
if (pMsg->pData) taosMemoryFree(pMsg->pData); if (pMsg->pData) taosMemoryFree(pMsg->pData);
if (pMsg->pEpSet) taosMemoryFree(pMsg->pEpSet);
if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) { if (code == TSDB_CODE_TMQ_CONSUMER_MISMATCH) {
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__RECOVER);
goto CREATE_MSG_FAIL; goto CREATE_MSG_FAIL;
@ -1239,8 +1240,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp);
tDecoderClear(&decoder); tDecoderClear(&decoder);
memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead));
} else {
ASSERT(0);
} }
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
@ -1365,6 +1364,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
} }
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED;
return -1; return -1;
} }
@ -1416,6 +1416,8 @@ END:
} else { } else {
taosMemoryFree(pParam); taosMemoryFree(pParam);
} }
taosMemoryFree(pMsg->pEpSet);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
return code; return code;
} }

7
source/common/src/tglobal.c
View File

@ -96,6 +96,7 @@ int32_t tsRedirectPeriod = 10;
int32_t tsRedirectFactor = 2; int32_t tsRedirectFactor = 2;
int32_t tsRedirectMaxPeriod = 1000; int32_t tsRedirectMaxPeriod = 1000;
int32_t tsMaxRetryWaitTime = 10000; int32_t tsMaxRetryWaitTime = 10000;
bool tsUseAdapter = false;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -201,9 +202,7 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
int32_t taosSetTfsCfg(SConfig *pCfg); int32_t taosSetTfsCfg(SConfig *pCfg);
#endif #endif
struct SConfig *taosGetCfg() { struct SConfig *taosGetCfg() { return tsCfg; }
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) { char *apolloUrl) {
@ -314,6 +313,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1;
if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = tsNumOfCores / 2;
tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
@ -668,6 +668,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32;
tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval;
tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
return 0; return 0;

1
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -174,6 +174,7 @@ typedef struct {
void* param; void* param;
char opername[TSDB_TRANS_OPER_LEN]; char opername[TSDB_TRANS_OPER_LEN];
SArray* pRpcArray; SArray* pRpcArray;
SRWLatch lockRpcArray;
} STrans; } STrans;
typedef struct { typedef struct {

41
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -133,7 +133,10 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
ASSERT(pConsumer); if (pConsumer == NULL) {
mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId);
return -1;
}
mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId, mInfo("receive consumer recover msg, consumer id %" PRId64 ", status %s", pRecoverMsg->consumerId,
mndConsumerStatusName(pConsumer->status)); mndConsumerStatusName(pConsumer->status));
@ -381,8 +384,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
return -1; return -1;
} }
ASSERT(strcmp(req.cgroup, pConsumer->cgroup) == 0);
atomic_store_32(&pConsumer->hbStatus, 0); atomic_store_32(&pConsumer->hbStatus, 0);
// 1. check consumer status // 1. check consumer status
@ -428,9 +429,8 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
for (int32_t i = 0; i < numOfTopics; i++) { for (int32_t i = 0; i < numOfTopics; i++) {
char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *topic = taosArrayGetP(pConsumer->currentTopics, i);
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic);
// txn guarantees pSub is created // txn guarantees pSub is created
ASSERT(pSub);
taosRLockLatch(&pSub->lock); taosRLockLatch(&pSub->lock);
SMqSubTopicEp topicEp = {0}; SMqSubTopicEp topicEp = {0};
@ -438,7 +438,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
// 2.1 fetch topic schema // 2.1 fetch topic schema
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
ASSERT(pTopic);
taosRLockLatch(&pTopic->lock); taosRLockLatch(&pTopic->lock);
tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN);
topicEp.schema.nCols = pTopic->schema.nCols; topicEp.schema.nCols = pTopic->schema.nCols;
@ -779,8 +778,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
taosWLockLatch(&pOldConsumer->lock); taosWLockLatch(&pOldConsumer->lock);
if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/
if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) {
pOldConsumer->status = MQ_CONSUMER_STATUS__READY; pOldConsumer->status = MQ_CONSUMER_STATUS__READY;
@ -802,8 +801,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
} }
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/
int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
/*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/ /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
@ -816,8 +815,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
@ -834,15 +833,15 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->rebalanceTime = pNewConsumer->upTime; pOldConsumer->rebalanceTime = pNewConsumer->upTime;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/
ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/
char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0));
// not exist in current topic // not exist in current topic
#if 1 #if 0
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) { for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->currentTopics); i++) {
char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); char *topic = taosArrayGetP(pOldConsumer->currentTopics, i);
ASSERT(strcmp(topic, addedTopic) != 0); A(strcmp(topic, addedTopic) != 0);
} }
#endif #endif
@ -883,15 +882,15 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
atomic_add_fetch_32(&pOldConsumer->epoch, 1); atomic_add_fetch_32(&pOldConsumer->epoch, 1);
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/
ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1); /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/
char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0);
// not exist in new topic // not exist in new topic
#if 1 #if 0
for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) {
char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i);
ASSERT(strcmp(topic, removedTopic) != 0); A(strcmp(topic, removedTopic) != 0);
} }
#endif #endif
@ -917,7 +916,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
} }
} }
// must find the topic // must find the topic
ASSERT(i < sz); /*A(i < sz);*/
// set status // set status
if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) { if (taosArrayGetSize(pOldConsumer->rebNewTopics) == 0 && taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0) {

51
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -115,13 +115,11 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
if (pStream->fixedSinkVgId == 0) { if (pStream->fixedSinkVgId == 0) {
SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb);
ASSERT(pDb);
if (pDb->cfg.numOfVgroups > 1) { if (pDb->cfg.numOfVgroups > 1) {
isShuffle = true; isShuffle = true;
pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH;
pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
ASSERT(0);
return -1; return -1;
} }
} }
@ -140,9 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
for (int32_t j = 0; j < sinkLvSize; j++) { for (int32_t j = 0; j < sinkLvSize; j++) {
SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
if (pLastLevelTask->nodeId == pVgInfo->vgId) { if (pLastLevelTask->nodeId == pVgInfo->vgId) {
ASSERT(pVgInfo->vgId > 0);
pVgInfo->taskId = pLastLevelTask->taskId; pVgInfo->taskId = pLastLevelTask->taskId;
ASSERT(pVgInfo->taskId != 0);
break; break;
} }
} }
@ -152,7 +148,6 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream
pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH;
SArray* pArray = taosArrayGetP(pStream->tasks, 0); SArray* pArray = taosArrayGetP(pStream->tasks, 0);
// one sink only // one sink only
ASSERT(taosArrayGetSize(pArray) == 1);
SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
@ -170,7 +165,6 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, co
plan->execNode.epSet = pTask->epSet; plan->execNode.epSet = pTask->epSet;
if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
ASSERT(0);
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
} }
@ -195,7 +189,6 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan,
plan->execNode.epSet = pTask->epSet; plan->execNode.epSet = pTask->epSet;
if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) {
ASSERT(0);
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
} }
@ -222,8 +215,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
void* pIter = NULL; void* pIter = NULL;
SArray* tasks = taosArrayGetP(pStream->tasks, 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0);
ASSERT(taosArrayGetSize(pStream->tasks) == 1);
while (1) { while (1) {
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@ -257,7 +248,10 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
pTask->tbSink.stbUid = pStream->targetStbUid; pTask->tbSink.stbUid = pStream->targetStbUid;
memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema);
ASSERT(pTask->tbSink.pSchemaWrapper); if (pTask->tbSink.pSchemaWrapper == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} }
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -265,7 +259,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
} }
int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) {
ASSERT(pStream->fixedSinkVgId != 0);
SArray* tasks = taosArrayGetP(pStream->tasks, 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0);
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewSStreamTask(pStream->uid);
if (pTask == NULL) { if (pTask == NULL) {
@ -275,8 +268,6 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) {
pTask->fillHistory = pStream->fillHistory; pTask->fillHistory = pStream->fillHistory;
mndAddTaskToTaskSet(tasks, pTask); mndAddTaskToTaskSet(tasks, pTask);
ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId);
pTask->nodeId = pStream->fixedSinkVgId; pTask->nodeId = pStream->fixedSinkVgId;
#if 0 #if 0
SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
@ -311,13 +302,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return -1; return -1;
} }
int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans);
ASSERT(planTotLevel <= 2);
pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*));
bool hasExtraSink = false; bool hasExtraSink = false;
bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0;
SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb);
ASSERT(pDbObj != NULL); if (pDbObj == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
bool multiTarget = pDbObj->cfg.numOfVgroups > 1; bool multiTarget = pDbObj->cfg.numOfVgroups > 1;
sdbRelease(pSdb, pDbObj); sdbRelease(pSdb, pDbObj);
@ -351,7 +345,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); if (plan->subplanType != SUBPLAN_TYPE_MERGE) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
pInnerTask = tNewSStreamTask(pStream->uid); pInnerTask = tNewSStreamTask(pStream->uid);
if (pInnerTask == NULL) { if (pInnerTask == NULL) {
@ -409,7 +406,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1);
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); if (plan->subplanType != SUBPLAN_TYPE_SCAN) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
@ -471,9 +471,15 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
taosArrayPush(pStream->tasks, &taskOneLevel); taosArrayPush(pStream->tasks, &taskOneLevel);
SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
ASSERT(LIST_LENGTH(inner->pNodeList) == 1); if (LIST_LENGTH(inner->pNodeList) != 1) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); if (plan->subplanType != SUBPLAN_TYPE_SCAN) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
@ -550,9 +556,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
} }
ASSERT(pSub->unassignedVgs);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
@ -590,10 +593,6 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
return 0; return 0;

15
source/dnode/mnode/impl/src/mndStream.c
View File

@ -326,13 +326,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
// deserialize ast // deserialize ast
if (nodesStringToNode(pObj->ast, &pAst) < 0) { if (nodesStringToNode(pObj->ast, &pAst) < 0) {
/*ASSERT(0);*/
goto FAIL; goto FAIL;
} }
// extract output schema from ast // extract output schema from ast
if (qExtractResultSchema(pAst, (int32_t *)&pObj->outputSchema.nCols, &pObj->outputSchema.pSchema) != 0) { if (qExtractResultSchema(pAst, (int32_t *)&pObj->outputSchema.nCols, &pObj->outputSchema.pSchema) != 0) {
/*ASSERT(0);*/
goto FAIL; goto FAIL;
} }
@ -347,13 +345,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
// using ast and param to build physical plan // using ast and param to build physical plan
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
/*ASSERT(0);*/
goto FAIL; goto FAIL;
} }
// save physcial plan // save physcial plan
if (nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL) != 0) { if (nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL) != 0) {
/*ASSERT(0);*/
goto FAIL; goto FAIL;
} }
@ -361,7 +357,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
if (pCreate->numOfTags) { if (pCreate->numOfTags) {
pObj->tagSchema.pSchema = taosMemoryCalloc(pCreate->numOfTags, sizeof(SSchema)); pObj->tagSchema.pSchema = taosMemoryCalloc(pCreate->numOfTags, sizeof(SSchema));
} }
ASSERT(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags)); /*A(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags));*/
for (int32_t i = 0; i < pCreate->numOfTags; i++) { for (int32_t i = 0; i < pCreate->numOfTags; i++) {
SField *pField = taosArrayGet(pCreate->pTags, i); SField *pField = taosArrayGet(pCreate->pTags, i);
pObj->tagSchema.pSchema[i].colId = pObj->outputSchema.nCols + i + 1; pObj->tagSchema.pSchema[i].colId = pObj->outputSchema.nCols + i + 1;
@ -378,9 +374,6 @@ FAIL:
} }
int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
if (pTask->taskLevel == TASK_LEVEL__AGG) {
ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0);
}
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, NULL, 0); tEncoderInit(&encoder, NULL, 0);
tEncodeSStreamTask(&encoder, pTask); tEncodeSStreamTask(&encoder, pTask);
@ -545,8 +538,6 @@ _OVER:
} }
static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
ASSERT(pTask->nodeId != 0);
// vnode // vnode
/*if (pTask->nodeId > 0) {*/ /*if (pTask->nodeId > 0) {*/
SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq));
@ -800,10 +791,9 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
int32_t sz = taosArrayGetSize(pLevel); int32_t sz = taosArrayGetSize(pLevel);
for (int32_t j = 0; j < sz; j++) { for (int32_t j = 0; j < sz; j++) {
SStreamTask *pTask = taosArrayGetP(pLevel, j); SStreamTask *pTask = taosArrayGetP(pLevel, j);
ASSERT(pTask->nodeId > 0); /*A(pTask->nodeId > 0);*/
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId);
if (pVgObj == NULL) { if (pVgObj == NULL) {
ASSERT(0);
taosRUnLockLatch(&pStream->lock); taosRUnLockLatch(&pStream->lock);
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -863,7 +853,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) {
ASSERT(0);
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
} }

12
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -628,6 +628,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction));
pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo));
taosInitRWLatch(&pTrans->lockRpcArray);
if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL || if (pTrans->redoActions == NULL || pTrans->undoActions == NULL || pTrans->commitActions == NULL ||
pTrans->pRpcArray == NULL) { pTrans->pRpcArray == NULL) {
@ -737,12 +738,14 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c
if (pTrans->oper == oper) { if (pTrans->oper == oper) {
if (strcasecmp(dbname, pTrans->dbname) == 0) { if (strcasecmp(dbname, pTrans->dbname) == 0) {
mInfo("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper); mInfo("trans:%d, db:%s oper:%d matched with input", pTrans->id, dbname, oper);
taosWLockLatch(&pTrans->lockRpcArray);
if (pTrans->pRpcArray == NULL) { if (pTrans->pRpcArray == NULL) {
pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->pRpcArray = taosArrayInit(4, sizeof(SRpcHandleInfo));
} }
if (pTrans->pRpcArray != NULL && taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) { if (pTrans->pRpcArray != NULL && taosArrayPush(pTrans->pRpcArray, &pMsg->info) != NULL) {
code = 0; code = 0;
} }
taosWUnLockLatch(&pTrans->lockRpcArray);
sdbRelease(pMnode->pSdb, pTrans); sdbRelease(pMnode->pSdb, pTrans);
break; break;
@ -944,8 +947,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
pTrans->failedTimes, code); pTrans->failedTimes, code);
} }
taosWLockLatch(&pTrans->lockRpcArray);
int32_t size = taosArrayGetSize(pTrans->pRpcArray); int32_t size = taosArrayGetSize(pTrans->pRpcArray);
if (size <= 0) return; if (size <= 0) {
taosWUnLockLatch(&pTrans->lockRpcArray);
return;
}
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i); SRpcHandleInfo *pInfo = taosArrayGet(pTrans->pRpcArray, i);
@ -997,6 +1004,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
} }
} }
taosArrayClear(pTrans->pRpcArray); taosArrayClear(pTrans->pRpcArray);
taosWUnLockLatch(&pTrans->lockRpcArray);
} }
int32_t mndTransProcessRsp(SRpcMsg *pRsp) { int32_t mndTransProcessRsp(SRpcMsg *pRsp) {

8
source/dnode/mnode/impl/src/mndUser.c
View File

@ -838,9 +838,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
if (pUser->superUser) { if (pUser->superUser) {
cols = 0; cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)userName, false); colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
char privilege[20] = {0}; char privilege[20] = {0};
@ -859,9 +859,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
char *db = taosHashIterate(pUser->readDbs, NULL); char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) { while (db != NULL) {
cols = 0; cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)userName, false); colDataAppend(pColInfo, numOfRows, (const char *)userName, false);
char privilege[20] = {0}; char privilege[20] = {0};

3
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -161,7 +161,10 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
SMetaSnapWriter* pWriter = *ppWriter; SMetaSnapWriter* pWriter = *ppWriter;
if (rollback) { if (rollback) {
metaInfo("vgId:%d, meta snapshot writer close and rollback start ", TD_VID(pWriter->pMeta->pVnode));
code = metaAbort(pWriter->pMeta); code = metaAbort(pWriter->pMeta);
metaInfo("vgId:%d, meta snapshot writer close and rollback finished, code:0x%x", TD_VID(pWriter->pMeta->pVnode),
code);
if (code) goto _err; if (code) goto _err;
} else { } else {
code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn); code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn);

3
source/dnode/vnode/src/meta/metaTable.c
View File

@ -710,6 +710,9 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) {
if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) {
return 0; return 0;
} }
metaDebug("vgId:%d, start to save ctime:%" PRId64 " uid:%" PRId64 " ct:%" PRId64, TD_VID(pMeta->pVnode), pME->version,
pME->uid, ctimeKey.ctime);
return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn);
} }

112
source/dnode/vnode/src/tq/tq.c
View File

@ -92,21 +92,21 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo);
if (tqMetaOpen(pTq) < 0) { if (tqMetaOpen(pTq) < 0) {
ASSERT(0); return NULL;
} }
pTq->pOffsetStore = tqOffsetOpen(pTq); pTq->pOffsetStore = tqOffsetOpen(pTq);
if (pTq->pOffsetStore == NULL) { if (pTq->pOffsetStore == NULL) {
ASSERT(0); return NULL;
} }
pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId);
if (pTq->pStreamMeta == NULL) { if (pTq->pStreamMeta == NULL) {
ASSERT(0); return NULL;
} }
if (streamLoadTasks(pTq->pStreamMeta) < 0) { if (streamLoadTasks(pTq->pStreamMeta) < 0) {
ASSERT(0); return NULL;
} }
return pTq; return pTq;
@ -166,19 +166,17 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq,
int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
SMqDataRsp* pRsp = &pPushEntry->dataRsp; SMqDataRsp* pRsp = &pPushEntry->dataRsp;
ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); #if 0
ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
ASSERT(!pRsp->withSchema); A(!pRsp->withSchema);
ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
/*if (pRsp->blockNum > 0) {*/ A(pRsp->rspOffset.version > pRsp->reqOffset.version);
/*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/
/*} else {*/
ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);
/*}*/
} }
#endif
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
@ -223,19 +221,21 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) {
} }
int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) {
ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); #if 0
ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
ASSERT(!pRsp->withSchema); A(!pRsp->withSchema);
ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); A(taosArrayGetSize(pRsp->blockSchema) == 0);
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
if (pRsp->blockNum > 0) { if (pRsp->blockNum > 0) {
ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); A(pRsp->rspOffset.version > pRsp->reqOffset.version);
} else { } else {
ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
} }
} }
#endif
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
@ -279,22 +279,24 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con
} }
int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) { int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) {
ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); #if 0
ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum);
A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum);
if (pRsp->withSchema) { if (pRsp->withSchema) {
ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum);
} else { } else {
ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); A(taosArrayGetSize(pRsp->blockSchema) == 0);
} }
if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) {
if (pRsp->blockNum > 0) { if (pRsp->blockNum > 0) {
ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); A(pRsp->rspOffset.version > pRsp->reqOffset.version);
} else { } else {
ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); A(pRsp->rspOffset.version >= pRsp->reqOffset.version);
} }
} }
#endif
int32_t len = 0; int32_t len = 0;
int32_t code = 0; int32_t code = 0;
@ -348,7 +350,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, msg, msgLen); tDecoderInit(&decoder, msg, msgLen);
if (tDecodeSTqOffset(&decoder, &offset) < 0) { if (tDecodeSTqOffset(&decoder, &offset) < 0) {
ASSERT(0);
return -1; return -1;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -362,8 +363,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m
if (offset.val.version + 1 == version) { if (offset.val.version + 1 == version) {
offset.val.version += 1; offset.val.version += 1;
} }
} else { /*} else {*/
ASSERT(0); /*A(0);*/
} }
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
if (pOffset != NULL && tqOffsetLessOrEqual(&offset, pOffset)) { if (pOffset != NULL && tqOffsetLessOrEqual(&offset, pOffset)) {
@ -371,7 +372,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t m
} }
if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -434,7 +434,7 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su
} }
#endif #endif
ASSERT(subType == TOPIC_SUB_TYPE__COLUMN); /*A(subType == TOPIC_SUB_TYPE__COLUMN);*/
pRsp->withSchema = false; pRsp->withSchema = false;
return 0; return 0;
@ -473,7 +473,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
// 1.find handle // 1.find handle
STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey));
/*ASSERT(pHandle);*/
if (pHandle == NULL) { if (pHandle == NULL) {
tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId, tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId,
TD_VID(pTq->pVnode), req.subKey); TD_VID(pTq->pVnode), req.subKey);
@ -560,7 +559,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType);
// lock // lock
taosWLockLatch(&pTq->pushLock); taosWLockLatch(&pTq->pushLock);
tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew); if (tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) {
return -1;
}
#if 1 #if 1
if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG &&
@ -599,7 +600,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
} }
// for taosx // for taosx
ASSERT(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN); /*A(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);*/
SMqMetaRsp metaRsp = {0}; SMqMetaRsp metaRsp = {0};
@ -607,7 +608,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
tqInitTaosxRsp(&taosxRsp, &req); tqInitTaosxRsp(&taosxRsp, &req);
if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { if (fetchOffsetNew.type != TMQ_OFFSET__LOG) {
tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) {
return -1;
}
if (metaRsp.metaRspLen > 0) { if (metaRsp.metaRspLen > 0) {
if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) {
@ -693,8 +696,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
} }
} else { } else {
ASSERT(pHandle->fetchMeta); /*A(pHandle->fetchMeta);*/
ASSERT(IS_META_MSG(pHead->msgType)); /*A(IS_META_MSG(pHead->msgType));*/
tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType);
tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer);
metaRsp.resMsgType = pHead->msgType; metaRsp.resMsgType = pHead->msgType;
@ -730,7 +733,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey));
if (pHandle) { if (pHandle) {
//walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId);
if (pHandle->pRef) { if (pHandle->pRef) {
walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId);
} }
@ -812,7 +815,6 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
// TODO version should be assigned and refed during preprocess // TODO version should be assigned and refed during preprocess
SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal); SWalRef* pRef = walRefCommittedVer(pTq->pVnode->pWal);
if (pRef == NULL) { if (pRef == NULL) {
ASSERT(0);
return -1; return -1;
} }
int64_t ver = pRef->refVer; int64_t ver = pRef->refVer;
@ -833,12 +835,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
pHandle->execHandle.task = pHandle->execHandle.task =
qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols, NULL); qCreateQueueExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle, &pHandle->execHandle.numOfCols, NULL);
ASSERT(pHandle->execHandle.task); /*A(pHandle->execHandle.task);*/
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(pHandle->execHandle.task, &scanner); qExtractStreamScanner(pHandle->execHandle.task, &scanner);
ASSERT(scanner); /*A(scanner);*/
pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
ASSERT(pHandle->execHandle.pExecReader); /*A(pHandle->execHandle.pExecReader);*/
} else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) {
pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); pHandle->pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode); pHandle->execHandle.pExecReader = tqOpenReader(pTq->pVnode);
@ -871,19 +873,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); taosHashPut(pTq->pHandle, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle));
tqDebug("try to persist handle %s consumer %" PRId64, req.subKey, pHandle->consumerId); tqDebug("try to persist handle %s consumer %" PRId64, req.subKey, pHandle->consumerId);
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
// TODO
ASSERT(0);
} }
} else { } else {
/*ASSERT(pExec->consumerId == req.oldConsumerId);*/
// TODO handle qmsg and exec modification // TODO handle qmsg and exec modification
atomic_store_32(&pHandle->epoch, -1); atomic_store_32(&pHandle->epoch, -1);
atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_store_64(&pHandle->consumerId, req.newConsumerId);
atomic_add_fetch_32(&pHandle->epoch, 1); atomic_add_fetch_32(&pHandle->epoch, 1);
taosMemoryFree(req.qmsg); taosMemoryFree(req.qmsg);
if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) {
// TODO
ASSERT(0);
} }
// close handle // close handle
} }
@ -892,9 +889,11 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
} }
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
#if 0
if (pTask->taskLevel == TASK_LEVEL__AGG) { if (pTask->taskLevel == TASK_LEVEL__AGG) {
ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); A(taosArrayGetSize(pTask->childEpInfo) != 0);
} }
#endif
pTask->refCnt = 1; pTask->refCnt = 1;
pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE;
@ -931,7 +930,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
}; };
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
ASSERT(pTask->exec.executor); if (pTask->exec.executor == NULL) {
return -1;
}
} else if (pTask->taskLevel == TASK_LEVEL__AGG) { } else if (pTask->taskLevel == TASK_LEVEL__AGG) {
pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
@ -944,7 +945,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
}; };
pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle);
ASSERT(pTask->exec.executor); if (pTask->exec.executor == NULL) {
return -1;
}
} }
// sink // sink
@ -956,8 +959,8 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->tbSink.vnode = pTq->pVnode; pTask->tbSink.vnode = pTq->pVnode;
pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2;
ASSERT(pTask->tbSink.pSchemaWrapper); /*A(pTask->tbSink.pSchemaWrapper);*/
ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/
pTask->tbSink.pTSchema = pTask->tbSink.pTSchema =
tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1);
@ -1007,7 +1010,8 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t len; int32_t len;
tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code);
if (code < 0) { if (code < 0) {
ASSERT(0); tqError("unable to encode rsp %d", __LINE__);
return -1;
} }
void* buf = rpcMallocCont(sizeof(SMsgHead) + len); void* buf = rpcMallocCont(sizeof(SMsgHead) + len);
((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId);
@ -1100,12 +1104,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
if (pTask == NULL) { if (pTask == NULL) {
return -1; return -1;
} }
ASSERT(pReq->taskId == pTask->taskId);
// check param // check param
int64_t fillVer1 = pTask->startVer; int64_t fillVer1 = pTask->startVer;
if (fillVer1 <= 0) { if (fillVer1 <= 0) {
ASSERT(0);
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return -1; return -1;
} }
@ -1300,7 +1302,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
} }
int32_t ref = atomic_sub_fetch_32(pRef, 1); int32_t ref = atomic_sub_fetch_32(pRef, 1);
ASSERT(ref >= 0); /*A(ref >= 0);*/
if (ref == 0) { if (ref == 0) {
blockDataDestroy(pDelBlock); blockDataDestroy(pDelBlock);
taosMemoryFree(pRef); taosMemoryFree(pRef);

27
source/dnode/vnode/src/tq/tqExec.c
View File

@ -29,7 +29,6 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t
int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols);
actualLen += sizeof(SRetrieveTableRsp); actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockDataLen, &actualLen);
taosArrayPush(pRsp->blockData, &buf); taosArrayPush(pRsp->blockData, &buf);
return 0; return 0;
@ -62,7 +61,6 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i
int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
const STqExecHandle* pExec = &pHandle->execHandle; const STqExecHandle* pExec = &pHandle->execHandle;
ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN);
qTaskInfo_t task = pExec->task; qTaskInfo_t task = pExec->task;
@ -87,7 +85,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
uint64_t ts = 0; uint64_t ts = 0;
tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId);
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0); tqError("vgId:%d task exec error since %s", pTq->pVnode->config.vgId, terrstr());
return -1;
} }
tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock);
@ -105,10 +104,14 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
} }
if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) {
ASSERT(0);
return -1; return -1;
} }
ASSERT(pRsp->rspOffset.type != 0);
if (pRsp->rspOffset.type == 0) {
tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts,
pRsp->rspOffset.uid, pRsp->rspOffset.version);
return -1;
}
ASSERT(pRsp->withTbName == false); ASSERT(pRsp->withTbName == false);
ASSERT(pRsp->withSchema == false); ASSERT(pRsp->withSchema == false);
@ -141,7 +144,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
uint64_t ts = 0; uint64_t ts = 0;
tqDebug("tmqsnap task start to execute"); tqDebug("tmqsnap task start to execute");
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0); tqError("vgId:%d task exec error since %s", pTq->pVnode->config.vgId, terrstr());
return -1;
} }
tqDebug("tmqsnap task execute end, get %p", pDataBlock); tqDebug("tmqsnap task execute end, get %p", pDataBlock);
@ -207,17 +211,20 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
break; break;
} }
if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { qStreamExtractOffset(task, &pRsp->rspOffset);
ASSERT(0);
if (pRsp->rspOffset.type == 0) {
tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts,
pRsp->rspOffset.uid, pRsp->rspOffset.version);
return -1;
} }
ASSERT(pRsp->rspOffset.type != 0);
return 0; return 0;
} }
int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) {
STqExecHandle* pExec = &pHandle->execHandle; STqExecHandle* pExec = &pHandle->execHandle;
ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN); /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/
SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
SArray* pSchemas = taosArrayInit(0, sizeof(void*)); SArray* pSchemas = taosArrayInit(0, sizeof(void*));

50
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -71,17 +71,14 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle) {
int32_t tqMetaOpen(STQ* pTq) { int32_t tqMetaOpen(STQ* pTq) {
if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) { if (tdbOpen(pTq->path, 16 * 1024, 1, &pTq->pMetaDB, 0) < 0) {
ASSERT(0);
return -1; return -1;
} }
if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) { if (tdbTbOpen("tq.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pExecStore, 0) < 0) {
ASSERT(0);
return -1; return -1;
} }
if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) { if (tdbTbOpen("tq.check.db", -1, -1, NULL, pTq->pMetaDB, &pTq->pCheckStore, 0) < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -135,19 +132,19 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) { 0) {
ASSERT(0); return -1;
} }
if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) {
/*ASSERT(0);*/ tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key);
} }
if (tdbCommit(pTq->pMetaDB, txn) < 0) { if (tdbCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
return 0; return 0;
@ -156,7 +153,6 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) {
int32_t tqMetaRestoreCheckInfo(STQ* pTq) { int32_t tqMetaRestoreCheckInfo(STQ* pTq) {
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) { if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -197,40 +193,42 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) {
int32_t code; int32_t code;
int32_t vlen; int32_t vlen;
tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code);
ASSERT(code == 0); if (code < 0) {
return -1;
}
tqDebug("tq save %s(%d) consumer %" PRId64 " vgId:%d", pHandle->subKey, (int32_t)strlen(pHandle->subKey), tqDebug("tq save %s(%d) consumer %" PRId64 " vgId:%d", pHandle->subKey, (int32_t)strlen(pHandle->subKey),
pHandle->consumerId, TD_VID(pTq->pVnode)); pHandle->consumerId, TD_VID(pTq->pVnode));
void* buf = taosMemoryCalloc(1, vlen); void* buf = taosMemoryCalloc(1, vlen);
if (buf == NULL) { if (buf == NULL) {
ASSERT(0); return -1;
} }
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, buf, vlen); tEncoderInit(&encoder, buf, vlen);
if (tEncodeSTqHandle(&encoder, pHandle) < 0) { if (tEncodeSTqHandle(&encoder, pHandle) < 0) {
ASSERT(0); return -1;
} }
TXN* txn; TXN* txn;
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) { 0) {
ASSERT(0); return -1;
} }
if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) { if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) {
ASSERT(0); return -1;
} }
if (tdbCommit(pTq->pMetaDB, txn) < 0) { if (tdbCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
@ -243,19 +241,18 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) <
0) { 0) {
ASSERT(0); return -1;
} }
if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) {
/*ASSERT(0);*/
} }
if (tdbCommit(pTq->pMetaDB, txn) < 0) { if (tdbCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { if (tdbPostCommit(pTq->pMetaDB, txn) < 0) {
ASSERT(0); return -1;
} }
return 0; return 0;
@ -264,7 +261,6 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) {
int32_t tqMetaRestoreHandle(STQ* pTq) { int32_t tqMetaRestoreHandle(STQ* pTq) {
TBC* pCur = NULL; TBC* pCur = NULL;
if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) {
ASSERT(0);
return -1; return -1;
} }
@ -284,7 +280,6 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
handle.pRef = walOpenRef(pTq->pVnode->pWal); handle.pRef = walOpenRef(pTq->pVnode->pWal);
if (handle.pRef == NULL) { if (handle.pRef == NULL) {
ASSERT(0);
return -1; return -1;
} }
walRefVer(handle.pRef, handle.snapshotVer); walRefVer(handle.pRef, handle.snapshotVer);
@ -300,12 +295,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
handle.execHandle.task = handle.execHandle.task =
qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL); qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL);
ASSERT(handle.execHandle.task); if (handle.execHandle.task == NULL) {
tqError("cannot create exec task for %s", handle.subKey);
return -1;
}
void* scanner = NULL; void* scanner = NULL;
qExtractStreamScanner(handle.execHandle.task, &scanner); qExtractStreamScanner(handle.execHandle.task, &scanner);
ASSERT(scanner); if (scanner == NULL) {
tqError("cannot extract stream scanner for %s", handle.subKey);
}
handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner);
ASSERT(handle.execHandle.pExecReader); if (handle.execHandle.pExecReader == NULL) {
tqError("cannot extract exec reader for %s", handle.subKey);
}
} else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) {
handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL);
handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode);

19
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -40,26 +40,23 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
if (code == 0) { if (code == 0) {
break; break;
} else { } else {
ASSERT(0); return -1;
// TODO handle error
} }
} }
int32_t size = htonl(head.size); int32_t size = htonl(head.size);
void* memBuf = taosMemoryCalloc(1, size); void* memBuf = taosMemoryCalloc(1, size);
if ((code = taosReadFile(pFile, memBuf, size)) != size) { if ((code = taosReadFile(pFile, memBuf, size)) != size) {
ASSERT(0); return -1;
// TODO handle error
} }
STqOffset offset; STqOffset offset;
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, memBuf, size); tDecoderInit(&decoder, memBuf, size);
if (tDecodeSTqOffset(&decoder, &offset) < 0) { if (tDecodeSTqOffset(&decoder, &offset) < 0) {
ASSERT(0); return -1;
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) {
ASSERT(0); return -1;
// TODO
} }
taosMemoryFree(memBuf); taosMemoryFree(memBuf);
} }
@ -85,7 +82,9 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) {
} }
char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); char* fname = tqOffsetBuildFName(pStore->pTq->path, 0);
if (tqOffsetRestoreFromFile(pStore, fname) < 0) { if (tqOffsetRestoreFromFile(pStore, fname) < 0) {
ASSERT(0); taosMemoryFree(fname);
taosMemoryFree(pStore);
return NULL;
} }
taosMemoryFree(fname); taosMemoryFree(fname);
return pStore; return pStore;
@ -124,7 +123,6 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
const char* sysErrStr = strerror(errno); const char* sysErrStr = strerror(errno);
tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname,
sysErrStr); sysErrStr);
ASSERT(0);
return -1; return -1;
} }
taosMemoryFree(fname); taosMemoryFree(fname);
@ -136,9 +134,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
int32_t bodyLen; int32_t bodyLen;
int32_t code; int32_t code;
tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code);
ASSERT(code == 0);
if (code < 0) { if (code < 0) {
ASSERT(0);
taosHashCancelIterate(pStore->pHash, pIter); taosHashCancelIterate(pStore->pHash, pIter);
return -1; return -1;
} }
@ -154,7 +150,6 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
// write file // write file
int64_t writeLen; int64_t writeLen;
if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) {
ASSERT(0);
tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen);
taosHashCancelIterate(pStore->pHash, pIter); taosHashCancelIterate(pStore->pHash, pIter);
taosMemoryFree(buf); taosMemoryFree(buf);

24
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -56,24 +56,28 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ);
if (pFile == NULL) { if (pFile == NULL) {
taosMemoryFree(fname); taosMemoryFree(fname);
return 0; return -1;
} }
int64_t sz = 0; int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL) < 0) { if (taosStatFile(fname, &sz, NULL) < 0) {
ASSERT(0); taosCloseFile(&pFile);
taosMemoryFree(fname);
return -1;
} }
taosMemoryFree(fname); taosMemoryFree(fname);
SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr)); SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr));
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
return terrno; return terrno;
} }
void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr)); void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr));
int64_t contLen = taosReadFile(pFile, abuf, sz); int64_t contLen = taosReadFile(pFile, abuf, sz);
if (contLen != sz) { if (contLen != sz) {
ASSERT(0); taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1; return -1;
} }
buf->size = sz; buf->size = sz;
@ -122,14 +126,17 @@ int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) {
if (rollback) { if (rollback) {
if (taosRemoveFile(pWriter->fname) < 0) { if (taosRemoveFile(pWriter->fname) < 0) {
ASSERT(0); taosMemoryFree(fname);
return -1;
} }
} else { } else {
if (taosRenameFile(pWriter->fname, fname) < 0) { if (taosRenameFile(pWriter->fname, fname) < 0) {
ASSERT(0); taosMemoryFree(fname);
return -1;
} }
if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) { if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) {
ASSERT(0); taosMemoryFree(fname);
return -1;
} }
} }
taosMemoryFree(fname); taosMemoryFree(fname);
@ -146,14 +153,13 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa
TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE);
SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
int64_t size = pHdr->size; int64_t size = pHdr->size;
ASSERT(size == nData - sizeof(SSnapDataHdr));
if (pFile) { if (pFile) {
int64_t contLen = taosWriteFile(pFile, pHdr->data, size); int64_t contLen = taosWriteFile(pFile, pHdr->data, size);
if (contLen != size) { if (contLen != size) {
ASSERT(0); taosCloseFile(&pFile);
return -1;
} }
} else { } else {
ASSERT(0);
return -1; return -1;
} }
return 0; return 0;

14
source/dnode/vnode/src/tq/tqPush.c
View File

@ -25,9 +25,7 @@ void tqTmrRspFunc(void* param, void* tmrId) {
static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) { static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) {
SStreamDataSubmit* pSubmit = *ppSubmit; SStreamDataSubmit* pSubmit = *ppSubmit;
while (pSubmit != NULL) { while (pSubmit != NULL) {
ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1);
if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) {
/*ASSERT(0);*/
} }
// update processed // update processed
atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver);
@ -160,8 +158,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
if (msgType == TDMT_VND_SUBMIT) { if (msgType == TDMT_VND_SUBMIT) {
tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId);
} else { } else {
// TODO tqError("tq push unexpected msg type %d", msgType);
ASSERT(0);
} }
if (rsp.blockNum == 0) { if (rsp.blockNum == 0) {
@ -169,9 +166,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
continue; continue;
} }
ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
rsp.rspOffset = fetchOffset; rsp.rspOffset = fetchOffset;
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
@ -270,7 +264,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
SSDataBlock* pDataBlock = NULL; SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0; uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) { if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0); tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr());
} }
if (pDataBlock == NULL) { if (pDataBlock == NULL) {
@ -289,7 +283,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
// remove from hash // remove from hash
size_t kLen; size_t kLen;
void* key = taosHashGetKey(pIter, &kLen); void* key = taosHashGetKey(pIter, &kLen);
void* keyCopy = taosMemoryMalloc(kLen); void* keyCopy = taosMemoryCalloc(1, kLen + 1);
memcpy(keyCopy, key, kLen); memcpy(keyCopy, key, kLen);
taosArrayPush(cachedKeys, &keyCopy); taosArrayPush(cachedKeys, &keyCopy);
@ -303,7 +297,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
void* key = taosArrayGetP(cachedKeys, i); void* key = taosArrayGetP(cachedKeys, i);
size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i); size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i);
if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) { if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) {
ASSERT(0); tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key);
} }
} }
taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree);

34
source/dnode/vnode/src/tq/tqRead.c
View File

@ -176,8 +176,6 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) {
goto end; goto end;
} }
realTbSuid = req.suid; realTbSuid = req.suid;
} else {
ASSERT(0);
} }
end: end:
@ -206,7 +204,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
code = walFetchBody(pHandle->pWalReader, ppCkHead); code = walFetchBody(pHandle->pWalReader, ppCkHead);
if (code < 0) { if (code < 0) {
ASSERT(0);
*fetchOffset = offset; *fetchOffset = offset;
code = -1; code = -1;
goto END; goto END;
@ -220,7 +217,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
if (IS_META_MSG(pHead->msgType)) { if (IS_META_MSG(pHead->msgType)) {
code = walFetchBody(pHandle->pWalReader, ppCkHead); code = walFetchBody(pHandle->pWalReader, ppCkHead);
if (code < 0) { if (code < 0) {
ASSERT(0);
*fetchOffset = offset; *fetchOffset = offset;
code = -1; code = -1;
goto END; goto END;
@ -238,7 +234,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea
} }
code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead); code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead);
if (code < 0) { if (code < 0) {
ASSERT(0);
*fetchOffset = offset; *fetchOffset = offset;
code = -1; code = -1;
goto END; goto END;
@ -297,11 +292,8 @@ void tqCloseReader(STqReader* pReader) {
int32_t tqSeekVer(STqReader* pReader, int64_t ver) { int32_t tqSeekVer(STqReader* pReader, int64_t ver) {
if (walReadSeekVer(pReader->pWalReader, ver) < 0) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) {
ASSERT(pReader->pWalReader->curInvalid);
ASSERT(pReader->pWalReader->curVersion == ver);
return -1; return -1;
} }
ASSERT(pReader->pWalReader->curVersion == ver);
return 0; return 0;
} }
@ -317,7 +309,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
ret->offset.version = pReader->ver; ret->offset.version = pReader->ver;
ret->fetchType = FETCH_TYPE__NONE; ret->fetchType = FETCH_TYPE__NONE;
tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version); tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version);
ASSERT(ret->offset.version >= 0);
return -1; return -1;
} }
void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead)); void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead));
@ -343,7 +334,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
memset(&ret->data, 0, sizeof(SSDataBlock)); memset(&ret->data, 0, sizeof(SSDataBlock));
int32_t code = tqRetrieveDataBlock2(&ret->data, pReader); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader);
if (code != 0 || ret->data.info.rows == 0) { if (code != 0 || ret->data.info.rows == 0) {
ASSERT(0);
continue; continue;
} }
ret->fetchType = FETCH_TYPE__DATA; ret->fetchType = FETCH_TYPE__DATA;
@ -354,7 +344,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) {
if (fromProcessedMsg) { if (fromProcessedMsg) {
ret->offset.type = TMQ_OFFSET__LOG; ret->offset.type = TMQ_OFFSET__LOG;
ret->offset.version = pReader->ver; ret->offset.version = pReader->ver;
ASSERT(pReader->ver >= 0);
ret->fetchType = FETCH_TYPE__SEP; ret->fetchType = FETCH_TYPE__SEP;
tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version);
return 0; return 0;
@ -518,7 +507,6 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) {
} }
if (pHandle->pBlock == NULL) return false; if (pHandle->pBlock == NULL) return false;
ASSERT(pHandle->tbIdHash == NULL);
void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t));
if (ret == NULL) { if (ret == NULL) {
return true; return true;
@ -784,7 +772,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
if (pReader->pSchema == NULL) { if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0; pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1; return -1;
@ -795,7 +782,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
if (pReader->pSchemaWrapper == NULL) { if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->msgIter.uid, pReader->cachedSchemaVer); pReader->msgIter.uid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0; pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1; return -1;
@ -898,7 +884,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
if (pReader->pSchema == NULL) { if (pReader->pSchema == NULL) {
tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table",
pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0; pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1; return -1;
@ -909,7 +894,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
if (pReader->pSchemaWrapper == NULL) { if (pReader->pSchemaWrapper == NULL) {
tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table",
pReader->msgIter.uid, pReader->cachedSchemaVer); pReader->msgIter.uid, pReader->cachedSchemaVer);
/*ASSERT(0);*/
pReader->cachedSchemaSuid = 0; pReader->cachedSchemaSuid = 0;
terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND;
return -1; return -1;
@ -1002,8 +986,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
break; break;
} }
ASSERT(sVal.valType != TD_VTYPE_NONE);
if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) {
goto FAIL; goto FAIL;
} }
@ -1075,8 +1057,6 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) {
} }
int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) {
ASSERT(pReader->tbIdHash != NULL);
for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) {
int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i);
taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)); taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t));
@ -1093,7 +1073,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
STqHandle* pExec = (STqHandle*)pIter; STqHandle* pExec = (STqHandle*)pIter;
if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd);
ASSERT(code == 0); if (code != 0) {
tqError("update qualified table error for %s", pExec->subKey);
continue;
}
} else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) {
if (!isAdd) { if (!isAdd) {
int32_t sz = taosArrayGetSize(tbUidList); int32_t sz = taosArrayGetSize(tbUidList);
@ -1112,7 +1095,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
int32_t code = metaGetTableEntryByUidCache(&mr, *id); int32_t code = metaGetTableEntryByUidCache(&mr, *id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno));
continue; continue;
} }
@ -1133,8 +1116,6 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
} else { } else {
// TODO handle delete table from stb // TODO handle delete table from stb
} }
} else {
ASSERT(0);
} }
} }
while (1) { while (1) {
@ -1143,7 +1124,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) {
SStreamTask* pTask = *(SStreamTask**)pIter; SStreamTask* pTask = *(SStreamTask**)pIter;
if (pTask->taskLevel == TASK_LEVEL__SOURCE) { if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd);
ASSERT(code == 0); if (code != 0) {
tqError("update qualified table error for stream task %d", pTask->taskId);
continue;
}
} }
} }
return 0; return 0;

10
source/dnode/vnode/src/tq/tqSink.c
View File

@ -19,7 +19,6 @@
int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock,
SBatchDeleteReq* deleteReq) { SBatchDeleteReq* deleteReq) {
ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT);
int32_t totRow = pDataBlock->info.rows; int32_t totRow = pDataBlock->info.rows;
SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -528,8 +527,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
int32_t code; int32_t code;
tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
if (code < 0) { if (code < 0) {
// terrno = TSDB_CODE_OUT_OF_MEMORY;
ASSERT(0); return;
} }
SEncoder encoder; SEncoder encoder;
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
@ -1000,7 +999,6 @@ void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data
tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size);
ASSERT(pTask->tbSink.pTSchema);
deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq));
SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true,
pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq); pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq);
@ -1011,10 +1009,6 @@ void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data
int32_t code; int32_t code;
int32_t len; int32_t len;
tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code);
if (code < 0) {
//
ASSERT(0);
}
SEncoder encoder; SEncoder encoder;
void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead));
void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead));

2
source/dnode/vnode/src/tq/tqSnapshot.c
View File

@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
} }
} }
ASSERT(pVal && vLen);
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;

3
source/dnode/vnode/src/tq/tqStreamStateSnap.c
View File

@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
} }
} }
ASSERT(pVal && vLen);
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -168,7 +166,6 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
if (rollback) { if (rollback) {
tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
ASSERT(0);
} else { } else {
code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err; if (code) goto _err;

11
source/dnode/vnode/src/tq/tqStreamTaskSnap.c
View File

@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
} }
} }
ASSERT(pVal && vLen);
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
if (*ppData == NULL) { if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
@ -146,7 +144,7 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
code = -1; code = -1;
taosMemoryFree(pWriter); taosMemoryFree(pWriter);
goto _err; goto _err;
@ -167,12 +165,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
STQ* pTq = pWriter->pTq; STQ* pTq = pWriter->pTq;
if (rollback) { if (rollback) {
tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn); tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
ASSERT(0);
} else { } else {
code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn); code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err; if (code) goto _err;
code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn); code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
if (code) goto _err; if (code) goto _err;
} }

2
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -1109,7 +1109,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) { if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) {
if (pWriter->dWriter.pWriter) { if (pWriter->dWriter.pWriter) {
ASSERT(fid > pWriter->fid); // ASSERT(fid > pWriter->fid);
code = tsdbSnapWriteCloseFile(pWriter); code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _err; if (code) goto _err;

10
source/libs/command/src/command.c
View File

@ -231,10 +231,6 @@ static const char* cacheModelStr(int8_t cacheModel) {
return TSDB_CACHE_MODEL_NONE_STR; return TSDB_CACHE_MODEL_NONE_STR;
} }
static const char* strictStr(int8_t strict) {
return TSDB_DB_STRICT_ON == strict ? TSDB_DB_STRICT_ON_STR : TSDB_DB_STRICT_OFF_STR;
}
static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) { static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) {
blockDataEnsureCapacity(pBlock, 1); blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1; pBlock->info.rows = 1;
@ -269,11 +265,11 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S
buf2 + VARSTR_HEADER_SIZE, buf2 + VARSTR_HEADER_SIZE,
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm " "CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " "WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"STRICT '%s' WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d", "WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d",
dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, strictStr(pCfg->strict), pCfg->walLevel, pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->walLevel, pCfg->numOfVgroups,
pCfg->numOfVgroups, 1 == pCfg->numOfStables); 1 == pCfg->numOfStables);
if (retentions) { if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);

38
source/libs/executor/src/scanoperator.c
View File

@ -1347,6 +1347,36 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
return code; return code;
} }
#if 0
void calBlockTag(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
SExprSupp* pTagCalSup = &pInfo->tagCalSup;
SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState;
if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return;
if (pBlock == NULL || pBlock->info.rows == 0) return;
void* tag = NULL;
int32_t tagLen = 0;
if (streamStateGetParTag(pState, pBlock->info.id.groupId, &tag, &tagLen) == 0) {
pBlock->info.tagLen = tagLen;
void* pTag = taosMemoryRealloc(pBlock->info.pTag, tagLen);
if (pTag == NULL) {
tdbFree(tag);
taosMemoryFree(pBlock->info.pTag);
pBlock->info.pTag = NULL;
pBlock->info.tagLen = 0;
return;
}
pBlock->info.pTag = pTag;
memcpy(pBlock->info.pTag, tag, tagLen);
tdbFree(tag);
return;
} else {
pBlock->info.pTag = NULL;
}
tdbFree(tag);
}
#endif
void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup;
SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState;
@ -1354,10 +1384,12 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
if (pBlock == NULL || pBlock->info.rows == 0) return; if (pBlock == NULL || pBlock->info.rows == 0) return;
void* tbname = NULL; void* tbname = NULL;
if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { if (streamStateGetParName(pState, pBlock->info.id.groupId, &tbname) == 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
tdbFree(tbname);
return;
} else {
pBlock->info.parTbName[0] = 0;
} }
tdbFree(tbname); tdbFree(tbname);

7
source/libs/executor/src/timesliceoperator.c
View File

@ -163,6 +163,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false); colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
continue; continue;
} else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) {
bool isFilled = true;
colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false);
continue;
} }
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
@ -274,6 +278,9 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp*
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
} else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) {
bool isFilled = false;
colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false);
} else { } else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);

23
source/libs/function/src/builtins.c
View File

@ -466,7 +466,7 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t
static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters // pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -480,14 +480,21 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return code; return code;
} }
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters // pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP};
return TSDB_CODE_SUCCESS;
}
static int32_t translateIsFilledPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes, .type = TSDB_DATA_TYPE_BOOL};
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3254,6 +3261,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = NULL, .sprocessFunc = NULL,
.finalizeFunc = NULL .finalizeFunc = NULL
}, },
{
.name = "_isfilled",
.type = FUNCTION_TYPE_ISFILLED,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC,
.translateFunc = translateIsFilledPseudoColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = NULL,
.finalizeFunc = NULL
},
{ {
.name = "_tags", .name = "_tags",
.type = FUNCTION_TYPE_TAGS, .type = FUNCTION_TYPE_TAGS,

1915
source/libs/nodes/src/nodesCodeFuncs.c
View File

File diff suppressed because it is too large Load Diff

16
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -305,8 +305,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SSetOperator)); return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return makeNode(type, sizeof(SSelectStmt)); return makeNode(type, sizeof(SSelectStmt));
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIFY_STMT:
return makeNode(type, sizeof(SVnodeModifOpStmt)); return makeNode(type, sizeof(SVnodeModifyOpStmt));
case QUERY_NODE_CREATE_DATABASE_STMT: case QUERY_NODE_CREATE_DATABASE_STMT:
return makeNode(type, sizeof(SCreateDatabaseStmt)); return makeNode(type, sizeof(SCreateDatabaseStmt));
case QUERY_NODE_DROP_DATABASE_STMT: case QUERY_NODE_DROP_DATABASE_STMT:
@ -321,8 +321,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateTableStmt)); return makeNode(type, sizeof(SCreateTableStmt));
case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: case QUERY_NODE_CREATE_SUBTABLE_CLAUSE:
return makeNode(type, sizeof(SCreateSubTableClause)); return makeNode(type, sizeof(SCreateSubTableClause));
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return makeNode(type, sizeof(SCreateMultiTableStmt)); return makeNode(type, sizeof(SCreateMultiTablesStmt));
case QUERY_NODE_DROP_TABLE_CLAUSE: case QUERY_NODE_DROP_TABLE_CLAUSE:
return makeNode(type, sizeof(SDropTableClause)); return makeNode(type, sizeof(SDropTableClause));
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
@ -818,8 +818,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode((SNode*)pStmt->pSlimit); nodesDestroyNode((SNode*)pStmt->pSlimit);
break; break;
} }
case QUERY_NODE_VNODE_MODIF_STMT: { case QUERY_NODE_VNODE_MODIFY_STMT: {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode;
destroyVgDataBlockArray(pStmt->pDataBlocks); destroyVgDataBlockArray(pStmt->pDataBlocks);
taosMemoryFreeClear(pStmt->pTableMeta); taosMemoryFreeClear(pStmt->pTableMeta);
taosHashCleanup(pStmt->pVgroupsHashObj); taosHashCleanup(pStmt->pVgroupsHashObj);
@ -862,8 +862,8 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode((SNode*)pStmt->pOptions); nodesDestroyNode((SNode*)pStmt->pOptions);
break; break;
} }
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
nodesDestroyList(((SCreateMultiTableStmt*)pNode)->pSubTables); nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables);
break; break;
case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field
break; break;

11
source/libs/parser/inc/sql.y
View File

@ -473,7 +473,15 @@ index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL
func_list(A) ::= func(B). { A = createNodeList(pCxt, B); } func_list(A) ::= func(B). { A = createNodeList(pCxt, B); }
func_list(A) ::= func_list(B) NK_COMMA func(C). { A = addNodeToList(pCxt, B, C); } func_list(A) ::= func_list(B) NK_COMMA func(C). { A = addNodeToList(pCxt, B, C); }
func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } func(A) ::= sma_func_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); }
%type sma_func_name { SToken }
%destructor sma_func_name { }
sma_func_name(A) ::= function_name(B). { A = B; }
sma_func_name(A) ::= COUNT(B). { A = B; }
sma_func_name(A) ::= FIRST(B). { A = B; }
sma_func_name(A) ::= LAST(B). { A = B; }
sma_func_name(A) ::= LAST_ROW(B). { A = B; }
sma_stream_opt(A) ::= . { A = createStreamOptions(pCxt); } sma_stream_opt(A) ::= . { A = createStreamOptions(pCxt); }
sma_stream_opt(A) ::= sma_stream_opt(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } sma_stream_opt(A) ::= sma_stream_opt(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; }
@ -733,6 +741,7 @@ pseudo_column(A) ::= WSTART(B).
pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= ISFILLED(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= QTAGS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QTAGS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }

8
source/libs/parser/src/parAstCreater.c
View File

@ -1208,7 +1208,7 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN
SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) { SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLE_STMT); SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLES_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
pStmt->pSubTables = pSubTables; pStmt->pSubTables = pSubTables;
return (SNode*)pStmt; return (SNode*)pStmt;
@ -1430,7 +1430,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST
} }
SCreateUserStmt* pStmt = (SCreateUserStmt*)nodesMakeNode(QUERY_NODE_CREATE_USER_STMT); SCreateUserStmt* pStmt = (SCreateUserStmt*)nodesMakeNode(QUERY_NODE_CREATE_USER_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
strcpy(pStmt->password, password); strcpy(pStmt->password, password);
pStmt->sysinfo = sysinfo; pStmt->sysinfo = sysinfo;
return (SNode*)pStmt; return (SNode*)pStmt;
@ -1443,7 +1443,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al
} }
SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT); SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
pStmt->alterType = alterType; pStmt->alterType = alterType;
switch (alterType) { switch (alterType) {
case TSDB_ALTER_USER_PASSWD: { case TSDB_ALTER_USER_PASSWD: {
@ -1474,7 +1474,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName) {
} }
SDropUserStmt* pStmt = (SDropUserStmt*)nodesMakeNode(QUERY_NODE_DROP_USER_STMT); SDropUserStmt* pStmt = (SDropUserStmt*)nodesMakeNode(QUERY_NODE_DROP_USER_STMT);
CHECK_OUT_OF_MEM(pStmt); CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName);
return (SNode*)pStmt; return (SNode*)pStmt;
} }

63
source/libs/parser/src/parAstParser.c
View File

@ -245,10 +245,14 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa
if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) { if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTags) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
} }
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
return code; return code;
} }
static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) { static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTablesStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, pStmt->pSubTables) { FOREACH(pNode, pStmt->pSubTables) {
@ -261,6 +265,10 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache);
} }
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
break; break;
} }
@ -351,38 +359,59 @@ static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateS
} }
static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowMnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowMnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MNODES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MNODES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowModules(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowModules(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MODULES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MODULES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowQnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowQnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_QNODES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_QNODES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowSnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowSnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_SNODES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_SNODES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowBnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowBnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_BNODES, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_BNODES,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowCluster(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowCluster(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CLUSTER, if (pCxt->pParseCxt->enableSysInfo) {
pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CLUSTER,
pCxt->pMetaCache);
}
return TSDB_CODE_SUCCESS;
} }
static int32_t collectMetaKeyFromShowDatabases(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowDatabases(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
@ -584,8 +613,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromFlushDatabase(pCxt, (SFlushDatabaseStmt*)pStmt); return collectMetaKeyFromFlushDatabase(pCxt, (SFlushDatabaseStmt*)pStmt);
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt); return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt);
case QUERY_NODE_ALTER_TABLE_STMT: case QUERY_NODE_ALTER_TABLE_STMT:

22
source/libs/parser/src/parAuthenticator.c
View File

@ -78,7 +78,7 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) {
} }
static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) {
if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) { if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->userName, TSDB_DEFAULT_USER)) {
return TSDB_CODE_PAR_PERMISSION_DENIED; return TSDB_CODE_PAR_PERMISSION_DENIED;
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -104,6 +104,22 @@ static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt)
return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ); return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ);
} }
static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE);
}
static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS;
SNode* pNode = NULL;
FOREACH(pNode, pStmt->pSubTables) {
code = checkAuth(pCxt, ((SCreateSubTableClause*)pNode)->dbName, AUTH_TYPE_WRITE);
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
return code;
}
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
@ -116,6 +132,10 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
return authDelete(pCxt, (SDeleteStmt*)pStmt); return authDelete(pCxt, (SDeleteStmt*)pStmt);
case QUERY_NODE_INSERT_STMT: case QUERY_NODE_INSERT_STMT:
return authInsert(pCxt, (SInsertStmt*)pStmt); return authInsert(pCxt, (SInsertStmt*)pStmt);
case QUERY_NODE_CREATE_TABLE_STMT:
return authCreateTable(pCxt, (SCreateTableStmt*)pStmt);
case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt);
case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT: case QUERY_NODE_SHOW_MODULES_STMT:

46
source/libs/parser/src/parInsertSml.c
View File

@ -158,11 +158,11 @@ end:
return code; return code;
} }
STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta){ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) {
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
SVCreateTbReq *pCreateTbReq = NULL; SVCreateTbReq* pCreateTbReq = NULL;
int ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
pTableMeta, &pCreateTbReq, &pTableCxt, false); sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
} }
@ -174,9 +174,9 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta){
return pTableCxt; return pTableCxt;
} }
int32_t smlBuildRow(STableDataCxt* pTableCxt){ int32_t smlBuildRow(STableDataCxt* pTableCxt) {
SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
if (TSDB_CODE_SUCCESS != ret) { if (TSDB_CODE_SUCCESS != ret) {
return ret; return ret;
} }
@ -184,12 +184,12 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt){
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *data, int32_t index){ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) {
int ret = TSDB_CODE_SUCCESS; int ret = TSDB_CODE_SUCCESS;
SSchema* pColSchema = schema + index; SSchema* pColSchema = schema + index;
SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
SSmlKv* kv = (SSmlKv*)data; SSmlKv* kv = (SSmlKv*)data;
if (kv->type == TSDB_DATA_TYPE_NCHAR){ if (kv->type == TSDB_DATA_TYPE_NCHAR) {
int32_t len = 0; int32_t len = 0;
char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE);
if (NULL == pUcs4) { if (NULL == pUcs4) {
@ -206,9 +206,9 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *data, int32
} }
pVal->value.pData = pUcs4; pVal->value.pData = pUcs4;
pVal->value.nData = len; pVal->value.nData = len;
} else if(kv->type == TSDB_DATA_TYPE_BINARY) { } else if (kv->type == TSDB_DATA_TYPE_BINARY) {
pVal->value.nData = kv->length; pVal->value.nData = kv->length;
pVal->value.pData = (uint8_t *)kv->value; pVal->value.pData = (uint8_t*)kv->value;
} else { } else {
memcpy(&pVal->value.val, &(kv->value), kv->length); memcpy(&pVal->value.val, &(kv->value), kv->length);
} }
@ -218,8 +218,9 @@ end:
return ret; return ret;
} }
int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
char* msgBuf, int16_t msgBufLen) {
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pTagsSchema = getTableTagSchema(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
@ -252,8 +253,9 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);
if(dataFormat){ if (dataFormat) {
STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
&pTableMeta->uid, sizeof(pTableMeta->uid));
if (NULL == pTableCxt) { if (NULL == pTableCxt) {
ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error");
goto end; goto end;
@ -267,7 +269,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
} }
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
ret = insGetTableDataCxt(((SVnodeModifOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error");
@ -300,11 +302,11 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc
for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
if (p == NULL) { if (p == NULL) {
continue; continue;
} }
SSmlKv *kv = *(SSmlKv **)p; SSmlKv* kv = *(SSmlKv**)p;
if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
@ -361,9 +363,9 @@ SQuery* smlInitHandle() {
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->haveResultSet = false; pQuery->haveResultSet = false;
pQuery->msgType = TDMT_VND_SUBMIT; pQuery->msgType = TDMT_VND_SUBMIT;
SVnodeModifOpStmt* stmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); SVnodeModifyOpStmt* stmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT);
if (NULL == stmt) { if (NULL == stmt) {
uError("create SVnodeModifOpStmt error"); uError("create SVnodeModifyOpStmt error");
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
return NULL; return NULL;
} }
@ -376,7 +378,7 @@ SQuery* smlInitHandle() {
} }
int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(handle)->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot;
// merge according to vgId // merge according to vgId
int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {

118
source/libs/parser/src/parInsertSql.c
View File

@ -155,7 +155,7 @@ static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) {
return code; return code;
} }
static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) { static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pDuplicate) {
*pDuplicate = false; *pDuplicate = false;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
@ -482,7 +482,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
// input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ...
// output pStmt->pSql: TAGS (tag1_value, ...) ... // output pStmt->pSql: TAGS (tag1_value, ...) ...
static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags); insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags);
SToken token; SToken token;
@ -496,7 +496,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt
return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags);
} }
static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken,
SArray* pTagName, SArray* pTagVals, STag** pTag) { SArray* pTagName, SArray* pTagVals, STag** pTag) {
if (!isNullValue(pTagSchema->type, pToken)) { if (!isNullValue(pTagSchema->type, pToken)) {
taosArrayPush(pTagName, pTagSchema->name); taosArrayPush(pTagName, pTagSchema->name);
@ -524,7 +524,7 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt
return code; return code;
} }
static int32_t buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) {
pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
if (NULL == pStmt->pCreateTblReq) { if (NULL == pStmt->pCreateTblReq) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -559,7 +559,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs
} }
// pSql -> tag1_value, ...) // pSql -> tag1_value, ...)
static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta); SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta);
SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal));
@ -617,7 +617,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt*
// input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ... // input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ...
// output pStmt->pSql: [table_options] ... // output pStmt->pSql: [table_options] ...
static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SToken token; SToken token;
NEXT_TOKEN(pStmt->pSql, token); NEXT_TOKEN(pStmt->pSql, token);
if (TK_TAGS != token.type) { if (TK_TAGS != token.type) {
@ -641,7 +641,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
return code; return code;
} }
static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->suid = pStmt->pTableMeta->uid;
pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->uid = pStmt->totalTbNum;
pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE;
@ -656,7 +656,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStm
return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES); return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES);
} }
static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
do { do {
int32_t index = 0; int32_t index = 0;
SToken token; SToken token;
@ -699,7 +699,7 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p
// output pStmt->pSql: // output pStmt->pSql:
// 1. [(field1_name, ...)] // 1. [(field1_name, ...)]
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
if (!pStmt->usingTableProcessing || pCxt->usingDuplicateTable) { if (!pStmt->usingTableProcessing || pCxt->usingDuplicateTable) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -773,7 +773,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt
return code; return code;
} }
static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SVgroupInfo vg; SVgroupInfo vg;
bool exists = true; bool exists = true;
@ -798,7 +798,7 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, boo
return code; return code;
} }
static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) {
SVgroupInfo vg; SVgroupInfo vg;
int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -810,7 +810,7 @@ static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt*
return code; return code;
} }
static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) {
SParseContext* pComCxt = pCxt->pComCxt; SParseContext* pComCxt = pCxt->pComCxt;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (pComCxt->async) { if (pComCxt->async) {
@ -836,25 +836,16 @@ static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) {
return taosHashPut(pDbs, dbFName, strlen(dbFName), dbFName, sizeof(dbFName)); return taosHashPut(pDbs, dbFName, strlen(dbFName), dbFName, sizeof(dbFName));
} }
static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
if (pCxt->forceUpdate) { if (pCxt->forceUpdate) {
pCxt->missCache = true; pCxt->missCache = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache);
#if 0
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache);
}
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache);
}
#else
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache);
} }
#endif
if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) {
code = collectUseDatabase(&pStmt->targetTableName, pStmt->pDbFNameHashObj); code = collectUseDatabase(&pStmt->targetTableName, pStmt->pDbFNameHashObj);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -864,11 +855,11 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt
return code; return code;
} }
static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) {
return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
} }
static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
if (pCxt->forceUpdate) { if (pCxt->forceUpdate) {
pCxt->missCache = true; pCxt->missCache = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -890,7 +881,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt*
return code; return code;
} }
static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SToken token; SToken token;
NEXT_TOKEN(pStmt->pSql, token); NEXT_TOKEN(pStmt->pSql, token);
int32_t code = preParseUsingTableName(pCxt, pStmt, &token); int32_t code = preParseUsingTableName(pCxt, pStmt, &token);
@ -909,7 +900,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpS
// output pStmt->pSql: // output pStmt->pSql:
// 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... // 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ...
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SToken token; SToken token;
int32_t index = 0; int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
@ -927,7 +918,7 @@ static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt*
return code; return code;
} }
static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) {
return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
} }
@ -938,7 +929,7 @@ static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpS
// output pStmt->pSql: // output pStmt->pSql:
// 1. [ USING ... ] ... // 1. [ USING ... ] ...
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SToken token; SToken token;
int32_t index = 0; int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
@ -952,7 +943,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif
return skipParentheses(pCxt, &pStmt->pSql); return skipParentheses(pCxt, &pStmt->pSql);
} }
static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt** pTableCxt) { static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) {
if (pCxt->pComCxt->async) { if (pCxt->pComCxt->async) {
return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid),
pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false);
@ -967,7 +958,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
&pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb);
} }
static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt) { static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) {
SToken token; SToken token;
int32_t index = 0; int32_t index = 0;
NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index);
@ -1010,7 +1001,8 @@ int32_t initTableColSubmitData(STableDataCxt* pTableCxt) {
// 1. [(tag1_name, ...)] ... // 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
// output pStmt->pSql: VALUES ... | FILE ... // output pStmt->pSql: VALUES ... | FILE ...
static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt** pTableCxt) { static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt,
STableDataCxt** pTableCxt) {
int32_t code = parseUsingClauseBottom(pCxt, pStmt); int32_t code = parseUsingClauseBottom(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = getTableDataCxt(pCxt, pStmt, pTableCxt); code = getTableDataCxt(pCxt, pStmt, pTableCxt);
@ -1028,7 +1020,7 @@ static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpS
// output pStmt->pSql: // output pStmt->pSql:
// 1. [(tag1_name, ...)] ... // 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) {
int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName); int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
// option: [(field1_name, ...)] // option: [(field1_name, ...)]
@ -1297,7 +1289,7 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC
} }
// pSql -> (field1_value, ...) [(field1_value2, ...) ...] // pSql -> (field1_value, ...) [(field1_value2, ...) ...]
static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt, static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt,
int32_t* pNumOfRows, SToken* pToken) { int32_t* pNumOfRows, SToken* pToken) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -1337,7 +1329,7 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
} }
// VALUES (field1_value, ...) [(field1_value2, ...) ...] // VALUES (field1_value, ...) [(field1_value2, ...) ...]
static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt, static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt,
SToken* pToken) { SToken* pToken) {
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken);
@ -1349,7 +1341,7 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* p
return code; return code;
} }
static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt, static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt,
int32_t* pNumOfRows) { int32_t* pNumOfRows) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
(*pNumOfRows) = 0; (*pNumOfRows) = 0;
@ -1401,7 +1393,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
return code; return code;
} }
static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt) { static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) {
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1417,7 +1409,7 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStm
return code; return code;
} }
static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath,
STableDataCxt* pTableCxt) { STableDataCxt* pTableCxt) {
char filePathStr[TSDB_FILENAME_LEN] = {0}; char filePathStr[TSDB_FILENAME_LEN] = {0};
if (TK_NK_STRING == pFilePath->type) { if (TK_NK_STRING == pFilePath->type) {
@ -1433,8 +1425,12 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* p
return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); return parseDataFromFileImpl(pCxt, pStmt, pTableCxt);
} }
static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt, static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt,
SToken* pToken) { SToken* pToken) {
if (tsUseAdapter) {
return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading");
}
NEXT_TOKEN(pStmt->pSql, *pToken); NEXT_TOKEN(pStmt->pSql, *pToken);
if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) {
return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z);
@ -1443,7 +1439,7 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
} }
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt* pTableCxt) { static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) {
SToken token; SToken token;
NEXT_TOKEN(pStmt->pSql, token); NEXT_TOKEN(pStmt->pSql, token);
switch (token.type) { switch (token.type) {
@ -1460,7 +1456,7 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
// input pStmt->pSql: // input pStmt->pSql:
// 1. [(tag1_name, ...)] ... // 1. [(tag1_name, ...)] ...
// 2. VALUES ... | FILE ... // 2. VALUES ... | FILE ...
static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1469,7 +1465,7 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod
return code; return code;
} }
static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
destroyBoundColInfo(&pCxt->tags); destroyBoundColInfo(&pCxt->tags);
taosMemoryFreeClear(pStmt->pTableMeta); taosMemoryFreeClear(pStmt->pTableMeta);
tdDestroySVCreateTbReq(pStmt->pCreateTblReq); tdDestroySVCreateTbReq(pStmt->pCreateTblReq);
@ -1482,7 +1478,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt
} }
// input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ...
static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) {
resetEnvPreTable(pCxt, pStmt); resetEnvPreTable(pCxt, pStmt);
int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName);
if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) {
@ -1491,7 +1487,7 @@ static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpSt
return code; return code;
} }
static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName, static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName,
bool* pHasData) { bool* pHasData) {
// no data in the sql string anymore. // no data in the sql string anymore.
if (0 == pTbName->n) { if (0 == pTbName->n) {
@ -1530,7 +1526,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags));
if (NULL == tags) { if (NULL == tags) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1548,7 +1544,7 @@ static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt)
return code; return code;
} }
static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) {
return setStmtInfo(pCxt, pStmt); return setStmtInfo(pCxt, pStmt);
} }
@ -1567,7 +1563,7 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStm
// [(field1_name, ...)] // [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...]; // [...];
static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
SToken token; SToken token;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
bool hasData = true; bool hasData = true;
@ -1590,7 +1586,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt
static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); }
static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT);
if (NULL == pStmt) { if (NULL == pStmt) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -1657,6 +1653,8 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet
if (1 != taosArrayGetSize(pTables)) { if (1 != taosArrayGetSize(pTables)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
taosMemoryFreeClear(*pMeta);
SMetaRes* pRes = taosArrayGet(pTables, 0); SMetaRes* pRes = taosArrayGet(pTables, 0);
if (TSDB_CODE_SUCCESS == pRes->code) { if (TSDB_CODE_SUCCESS == pRes->code) {
*pMeta = tableMetaDup((const STableMeta*)pRes->pRes); *pMeta = tableMetaDup((const STableMeta*)pRes->pRes);
@ -1667,7 +1665,7 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet
return pRes->code; return pRes->code;
} }
static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) { static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) {
if (1 != taosArrayGetSize(pTables)) { if (1 != taosArrayGetSize(pTables)) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -1686,7 +1684,7 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt
} }
static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData,
SVnodeModifOpStmt* pStmt, bool isStb) { SVnodeModifyOpStmt* pStmt, bool isStb) {
int32_t code = checkAuthFromMetaData(pMetaData->pUser); int32_t code = checkAuthFromMetaData(pMetaData->pUser);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta);
@ -1719,7 +1717,7 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) {
} }
static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData,
SVnodeModifOpStmt* pStmt) { SVnodeModifyOpStmt* pStmt) {
clearCatalogReq(pCatalogReq); clearCatalogReq(pCatalogReq);
if (pStmt->usingTableProcessing) { if (pStmt->usingTableProcessing) {
@ -1733,7 +1731,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery)
int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
(*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj,
&pStmt->pTableBlockHashObj); &pStmt->pTableBlockHashObj);
@ -1762,7 +1760,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR
return resetVnodeModifOpStmt(pCxt, *pQuery); return resetVnodeModifOpStmt(pCxt, *pQuery);
} }
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(*pQuery)->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(*pQuery)->pRoot;
if (!pStmt->fileProcessing) { if (!pStmt->fileProcessing) {
return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, pStmt); return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, pStmt);
@ -1772,7 +1770,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR
} }
static int32_t setRefreshMate(SQuery* pQuery) { static int32_t setRefreshMate(SQuery* pQuery) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) {
taosArrayDestroy(pQuery->pTableList); taosArrayDestroy(pQuery->pTableList);
@ -1803,7 +1801,7 @@ static int32_t setRefreshMate(SQuery* pQuery) {
// [(field1_name, ...)] // [(field1_name, ...)]
// VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path
// [...]; // [...];
static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg); int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(pCxt, pStmt); code = parseInsertBody(pCxt, pStmt);
@ -1811,7 +1809,7 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpS
return code; return code;
} }
static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1829,7 +1827,7 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStm
return code; return code;
} }
static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
int32_t code = parseInsertTableClauseBottom(pCxt, pStmt); int32_t code = parseInsertTableClauseBottom(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = parseInsertBody(pCxt, pStmt); code = parseInsertBody(pCxt, pStmt);
@ -1837,7 +1835,7 @@ static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpS
return code; return code;
} }
static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) {
if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) { if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) {
return parseInsertSqlFromStart(pCxt, pStmt); return parseInsertSqlFromStart(pCxt, pStmt);
} }
@ -1889,7 +1887,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray**
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) { static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) {
int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (0 == pStmt->usingTableName.type) { if (0 == pStmt->usingTableName.type) {
@ -1905,7 +1903,7 @@ static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStm
} }
static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) {
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
if (pCxt->missCache) { if (pCxt->missCache) {
parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted before cache miss", pCxt->pComCxt->requestId, parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted before cache miss", pCxt->pComCxt->requestId,
pStmt->totalRowsNum, pStmt->totalTbNum); pStmt->totalRowsNum, pStmt->totalTbNum);
@ -1930,7 +1928,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal
int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = setNextStageInfo(&context, *pQuery, pCatalogReq); code = setNextStageInfo(&context, *pQuery, pCatalogReq);

84
source/libs/parser/src/parInsertStmt.c
View File

@ -29,13 +29,13 @@ typedef struct SKvParam {
char buf[TSDB_MAX_TAGS_LEN]; char buf[TSDB_MAX_TAGS_LEN];
} SKvParam; } SKvParam;
int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData) { int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) {
*pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData));
if (NULL == *pData) { if (NULL == *pData) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
SSubmitTbData *pNew = *pData; SSubmitTbData* pNew = *pData;
*pNew = *pDataBlock->pData; *pNew = *pDataBlock->pData;
@ -44,18 +44,17 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pDa
int32_t colNum = taosArrayGetSize(pNew->aCol); int32_t colNum = taosArrayGetSize(pNew->aCol);
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
SColData *pCol = (SColData*)taosArrayGet(pNew->aCol, i); SColData* pCol = (SColData*)taosArrayGet(pNew->aCol, i);
tColDataDeepClear(pCol); tColDataDeepClear(pCol);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* pVgDataBlocks = NULL; SArray* pVgDataBlocks = NULL;
SVnodeModifOpStmt *pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
// merge according to vgId // merge according to vgId
if (taosHashGetSize(pBlockHash) > 0) { if (taosHashGetSize(pBlockHash) > 0) {
@ -73,9 +72,9 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash
int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SBoundColInfo* tags = (SBoundColInfo*)boundTags; SBoundColInfo* tags = (SBoundColInfo*)boundTags;
if (NULL == tags) { if (NULL == tags) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -172,7 +171,8 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch
} }
} }
insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName,
pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL);
end: end:
for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
@ -212,7 +212,8 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
continue; continue;
} }
if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i],
(TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) {
if (errno == E2BIG) { if (errno == E2BIG) {
return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name);
} }
@ -232,17 +233,17 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND*
} }
int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
int32_t rowNum = bind->num; int32_t rowNum = bind->num;
TAOS_MULTI_BIND ncharBind = {0}; TAOS_MULTI_BIND ncharBind = {0};
TAOS_MULTI_BIND* pBind = NULL; TAOS_MULTI_BIND* pBind = NULL;
int32_t code = 0; int32_t code = 0;
for (int c = 0; c < boundInfo->numOfBound; ++c) { for (int c = 0; c < boundInfo->numOfBound; ++c) {
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c); SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c);
if (bind[c].num != rowNum) { if (bind[c].num != rowNum) {
@ -280,15 +281,15 @@ _return:
int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t rowNum) { int32_t rowNum) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta);
SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo;
SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]]; SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]];
SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx); SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx);
TAOS_MULTI_BIND ncharBind = {0}; TAOS_MULTI_BIND ncharBind = {0};
TAOS_MULTI_BIND* pBind = NULL; TAOS_MULTI_BIND* pBind = NULL;
int32_t code = 0; int32_t code = 0;
if (bind->num != rowNum) { if (bind->num != rowNum) {
return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
@ -320,8 +321,8 @@ _return:
return code; return code;
} }
int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum,
uint8_t timePrec) { TAOS_FIELD_E** fields, uint8_t timePrec) {
if (fields) { if (fields) {
*fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E));
if (NULL == *fields) { if (NULL == *fields) {
@ -347,7 +348,7 @@ int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSc
} }
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) {
STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; STableDataCxt* pDataBlock = (STableDataCxt*)pBlock;
SBoundColInfo* tags = (SBoundColInfo*)boundTags; SBoundColInfo* tags = (SBoundColInfo*)boundTags;
if (NULL == tags) { if (NULL == tags) {
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;
@ -382,18 +383,18 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, fieldNum, fields, CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema,
pDataBlock->pMeta->tableInfo.precision)); fieldNum, fields, pDataBlock->pMeta->tableInfo.precision));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) {
STableDataCxt* pBlock = (STableDataCxt*)block; STableDataCxt* pBlock = (STableDataCxt*)block;
int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); int32_t colNum = taosArrayGetSize(pBlock->pData->aCol);
for (int32_t i = 0; i < colNum; ++i) { for (int32_t i = 0; i < colNum; ++i) {
SColData *pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i);
if (deepClear) { if (deepClear) {
tColDataDeepClear(pCol); tColDataDeepClear(pCol);
} else { } else {
@ -436,12 +437,13 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese
insDestroyTableDataCxt(*pDst); insDestroyTableDataCxt(*pDst);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex, pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex,
pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex));
pNewCxt->boundColsInfo.pColIndex = pNewColIdx; pNewCxt->boundColsInfo.pColIndex = pNewColIdx;
} }
if (pCxt->pData) { if (pCxt->pData) {
SSubmitTbData *pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData)); SSubmitTbData* pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData));
if (NULL == pNewTb) { if (NULL == pNewTb) {
insDestroyTableDataCxt(*pDst); insDestroyTableDataCxt(*pDst);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -463,11 +465,11 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese
} }
} }
return code; return code;
} }
int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb) { int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId,
bool rebuildCreateTb) {
int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); int32_t code = qCloneStmtDataBlock(pDst, pSrc, false);
if (code) { if (code) {
return code; return code;

34
source/libs/parser/src/parInsertUtil.c
View File

@ -206,7 +206,7 @@ void qDestroyBoundColInfo(void* pInfo) {
} }
static int32_t createTableDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, static int32_t createTableDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta,
STableDataBlocks** dataBlocks) { STableDataBlocks** dataBlocks) {
STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks));
if (dataBuf == NULL) { if (dataBuf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1373,7 +1373,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
return code; return code;
} }
static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD *fields, int numFields){ static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) {
bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool));
if (NULL == pUseCols) { if (NULL == pUseCols) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1383,7 +1383,7 @@ static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD *f
int16_t lastColIdx = -1; // last column found int16_t lastColIdx = -1; // last column found
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
for(int i = 0; i < numFields; i++) { for (int i = 0; i < numFields; i++) {
SToken token; SToken token;
token.z = fields[i].name; token.z = fields[i].name;
token.n = strlen(fields[i].name); token.n = strlen(fields[i].name);
@ -1418,15 +1418,16 @@ static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD *f
return code; return code;
} }
int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq *pCreateTb, TAOS_FIELD *tFields, int numFields){ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields,
int numFields) {
STableDataCxt* pTableCxt = NULL; STableDataCxt* pTableCxt = NULL;
int ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
pTableMeta, &pCreateTb, &pTableCxt, true); sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("insGetTableDataCxt error"); uError("insGetTableDataCxt error");
goto end; goto end;
} }
if (tFields != NULL){ if (tFields != NULL) {
ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError("bindFileds error"); uError("bindFileds error");
@ -1436,12 +1437,13 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreate
// no need to bind, because select * get all fields // no need to bind, because select * get all fields
ret = initTableColSubmitData(pTableCxt); ret = initTableColSubmitData(pTableCxt);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
uError( "initTableColSubmitData error"); uError("initTableColSubmitData error");
goto end; goto end;
} }
char* p = (char*)data; char* p = (char*)data;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
// length |
p += sizeof(int32_t); p += sizeof(int32_t);
p += sizeof(int32_t); p += sizeof(int32_t);
@ -1454,7 +1456,7 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreate
p += sizeof(int32_t); p += sizeof(int32_t);
p += sizeof(uint64_t); p += sizeof(uint64_t);
int8_t *fields = p; int8_t* fields = p;
p += numOfCols * (sizeof(int8_t) + sizeof(int32_t)); p += numOfCols * (sizeof(int8_t) + sizeof(int32_t));
int32_t* colLength = (int32_t*)p; int32_t* colLength = (int32_t*)p;
@ -1462,20 +1464,20 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreate
char* pStart = p; char* pStart = p;
SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta);
SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo;
if(boundInfo->numOfBound != numOfCols){ if (boundInfo->numOfBound != numOfCols) {
uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols);
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
for (int c = 0; c < boundInfo->numOfBound; ++c) { for (int c = 0; c < boundInfo->numOfBound; ++c) {
SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]];
SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c);
if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
uError( "type or bytes not equal"); uError("type or bytes not equal");
ret = TSDB_CODE_INVALID_PARA; ret = TSDB_CODE_INVALID_PARA;
goto end; goto end;
} }
@ -1487,7 +1489,7 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreate
} else { } else {
pStart += BitmapLen(numOfRows); pStart += BitmapLen(numOfRows);
} }
char *pData = pStart; char* pData = pStart;
tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData);
fields += sizeof(int8_t) + sizeof(int32_t); fields += sizeof(int8_t) + sizeof(int32_t);

1
source/libs/parser/src/parTokenizer.c
View File

@ -260,6 +260,7 @@ static SKeyword keywordTable[] = {
{"WRITE", TK_WRITE}, {"WRITE", TK_WRITE},
{"_C0", TK_ROWTS}, {"_C0", TK_ROWTS},
{"_IROWTS", TK_IROWTS}, {"_IROWTS", TK_IROWTS},
{"_ISFILLED", TK_ISFILLED},
{"_QDURATION", TK_QDURATION}, {"_QDURATION", TK_QDURATION},
{"_QEND", TK_QEND}, {"_QEND", TK_QEND},
{"_QSTART", TK_QSTART}, {"_QSTART", TK_QSTART},

18
source/libs/parser/src/parTranslater.c
View File

@ -352,7 +352,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa
code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta);
} }
} }
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code && TSDB_CODE_TSC_INVALID_TABLE_NAME != code) {
parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
tstrerror(code), pName->dbname, pName->tname); tstrerror(code), pName->dbname, pName->tname);
} }
@ -5083,7 +5083,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p
static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) {
SCreateUserReq createReq = {0}; SCreateUserReq createReq = {0};
strcpy(createReq.user, pStmt->useName); strcpy(createReq.user, pStmt->userName);
createReq.createType = 0; createReq.createType = 0;
createReq.superUser = 0; createReq.superUser = 0;
createReq.sysInfo = pStmt->sysinfo; createReq.sysInfo = pStmt->sysinfo;
@ -5095,7 +5095,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt
static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
strcpy(alterReq.user, pStmt->useName); strcpy(alterReq.user, pStmt->userName);
alterReq.alterType = pStmt->alterType; alterReq.alterType = pStmt->alterType;
alterReq.superUser = 0; alterReq.superUser = 0;
alterReq.enable = pStmt->enable; alterReq.enable = pStmt->enable;
@ -5110,7 +5110,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt
static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) { static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) {
SDropUserReq dropReq = {0}; SDropUserReq dropReq = {0};
strcpy(dropReq.user, pStmt->useName); strcpy(dropReq.user, pStmt->userName);
return buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq); return buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq);
} }
@ -6645,7 +6645,7 @@ static void destroyCreateTbReqBatch(void* data) {
} }
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) {
SVnodeModifOpStmt* pNewStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); SVnodeModifyOpStmt* pNewStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT);
if (pNewStmt == NULL) { if (pNewStmt == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -7029,7 +7029,7 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) {
} }
static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) {
SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)pQuery->pRoot;
SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (NULL == pVgroupHashmap) { if (NULL == pVgroupHashmap) {
@ -7637,7 +7637,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
code = rewriteCreateTable(pCxt, pQuery); code = rewriteCreateTable(pCxt, pQuery);
} }
break; break;
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
code = rewriteCreateMultiTable(pCxt, pQuery); code = rewriteCreateMultiTable(pCxt, pQuery);
break; break;
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
@ -7734,9 +7734,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->msgType = TDMT_VND_SUBMIT; pQuery->msgType = TDMT_VND_SUBMIT;
break; break;
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIFY_STMT:
pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
pQuery->msgType = toMsgType(((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType); pQuery->msgType = toMsgType(((SVnodeModifyOpStmt*)pQuery->pRoot)->sqlNodeType);
break; break;
case QUERY_NODE_DESCRIBE_STMT: case QUERY_NODE_DESCRIBE_STMT:
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:

6458
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/parser/test/parAlterToBalanceTest.cpp
View File

@ -504,8 +504,8 @@ TEST_F(ParserInitialATest, alterTable) {
}; };
setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) {
ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIF_STMT); ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIFY_STMT);
SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot;
ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT);
ASSERT_NE(pStmt->pDataBlocks, nullptr); ASSERT_NE(pStmt->pDataBlocks, nullptr);

12
source/libs/planner/src/planLogicCreater.c
View File

@ -953,6 +953,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} }
if (TSDB_CODE_SUCCESS == code && 0 == LIST_LENGTH(pFill->node.pTargets)) {
code = createColumnByRewriteExpr(pFill->pWStartTs, &pFill->node.pTargets);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pFill; *pLogicNode = (SLogicNode*)pFill;
} else { } else {
@ -1382,7 +1386,7 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator*
static int32_t getMsgType(ENodeType sqlType) { static int32_t getMsgType(ENodeType sqlType) {
switch (sqlType) { switch (sqlType) {
case QUERY_NODE_CREATE_TABLE_STMT: case QUERY_NODE_CREATE_TABLE_STMT:
case QUERY_NODE_CREATE_MULTI_TABLE_STMT: case QUERY_NODE_CREATE_MULTI_TABLES_STMT:
return TDMT_VND_CREATE_TABLE; return TDMT_VND_CREATE_TABLE;
case QUERY_NODE_DROP_TABLE_STMT: case QUERY_NODE_DROP_TABLE_STMT:
return TDMT_VND_DROP_TABLE; return TDMT_VND_DROP_TABLE;
@ -1396,7 +1400,7 @@ static int32_t getMsgType(ENodeType sqlType) {
return TDMT_VND_SUBMIT; return TDMT_VND_SUBMIT;
} }
static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifyOpStmt* pStmt, SLogicNode** pLogicNode) {
SVnodeModifyLogicNode* pModif = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); SVnodeModifyLogicNode* pModif = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY);
if (NULL == pModif) { if (NULL == pModif) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1580,8 +1584,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode); return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode);
case QUERY_NODE_VNODE_MODIF_STMT: case QUERY_NODE_VNODE_MODIFY_STMT:
return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); return createVnodeModifLogicNode(pCxt, (SVnodeModifyOpStmt*)pStmt, pLogicNode);
case QUERY_NODE_EXPLAIN_STMT: case QUERY_NODE_EXPLAIN_STMT:
return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode); return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode);
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:

2
source/libs/planner/test/planBasicTest.cpp
View File

@ -104,6 +104,8 @@ TEST_F(PlanBasicTest, interpFunc) {
run("SELECT _IROWTS, INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); run("SELECT _IROWTS, INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
run("SELECT _IROWTS, INTERP(c1), _ISFILLED FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
run("SELECT TBNAME, _IROWTS, INTERP(c1) FROM t1 PARTITION BY TBNAME " run("SELECT TBNAME, _IROWTS, INTERP(c1) FROM t1 PARTITION BY TBNAME "
"RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); "RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)");
} }

1
source/libs/scheduler/src/schRemote.c
View File

@ -461,6 +461,7 @@ int32_t schHandleLinkBrokenCallback(void *param, SDataBuf *pMsg, int32_t code) {
if (head->isHbParam) { if (head->isHbParam) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
taosMemoryFree(pMsg->pEpSet);
SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param; SSchHbCallbackParam *hbParam = (SSchHbCallbackParam *)param;
SSchTrans trans = {.pTrans = hbParam->pTrans, .pHandle = NULL}; SSchTrans trans = {.pTrans = hbParam->pTrans, .pHandle = NULL};

7
source/libs/stream/src/streamExec.c
View File

@ -122,7 +122,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
block.info.childId = pTask->selfChildId; block.info.childId = pTask->selfChildId;
taosArrayPush(pRes, &block); taosArrayPush(pRes, &block);
if (++batchCnt >= batchSz) break; batchCnt++;
qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz);
if (batchCnt >= batchSz) break;
} }
if (taosArrayGetSize(pRes) == 0) { if (taosArrayGetSize(pRes) == 0) {
taosArrayDestroy(pRes); taosArrayDestroy(pRes);
@ -140,6 +144,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
streamTaskOutput(pTask, qRes); streamTaskOutput(pTask, qRes);
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt);
streamDispatch(pTask); streamDispatch(pTask);
} }
if (finished) break; if (finished) break;

20
source/libs/stream/src/streamState.c
View File

@ -159,6 +159,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int
goto _err; goto _err;
} }
if (tdbTbOpen("partag.state.db", sizeof(int64_t), -1, NULL, pState->pTdbState->db, &pState->pTdbState->pParTagDb, 0) <
0) {
goto _err;
}
if (streamStateBegin(pState) < 0) { if (streamStateBegin(pState) < 0) {
goto _err; goto _err;
} }
@ -173,6 +178,7 @@ _err:
tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pFillStateDb);
tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb);
tdbTbClose(pState->pTdbState->pParNameDb); tdbTbClose(pState->pTdbState->pParNameDb);
tdbTbClose(pState->pTdbState->pParTagDb);
tdbClose(pState->pTdbState->db); tdbClose(pState->pTdbState->db);
streamStateDestroy(pState); streamStateDestroy(pState);
return NULL; return NULL;
@ -186,6 +192,7 @@ void streamStateClose(SStreamState* pState) {
tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pFillStateDb);
tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb);
tdbTbClose(pState->pTdbState->pParNameDb); tdbTbClose(pState->pTdbState->pParNameDb);
tdbTbClose(pState->pTdbState->pParTagDb);
tdbClose(pState->pTdbState->db); tdbClose(pState->pTdbState->db);
streamStateDestroy(pState); streamStateDestroy(pState);
@ -821,10 +828,17 @@ _end:
return res; return res;
} }
int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen) {
return tdbTbUpsert(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tag, tagLen, pState->pTdbState->txn);
}
int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen) {
return tdbTbGet(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tagVal, tagLen);
}
int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) {
tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN,
pState->pTdbState->txn); pState->pTdbState->txn);
return 0;
} }
int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) { int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) {

24
source/libs/sync/src/syncPipeline.c
View File

@ -275,6 +275,8 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
SyncIndex index = pEntry->index; SyncIndex index = pEntry->index;
SyncIndex prevIndex = pEntry->index - 1; SyncIndex prevIndex = pEntry->index - 1;
SyncTerm lastMatchTerm = syncLogBufferGetLastMatchTerm(pBuf); SyncTerm lastMatchTerm = syncLogBufferGetLastMatchTerm(pBuf);
SSyncRaftEntry* pExist = NULL;
bool inBuf = true;
if (index <= pBuf->commitIndex) { if (index <= pBuf->commitIndex) {
sTrace("vgId:%d, already committed. index: %" PRId64 ", term: %" PRId64 ". log buffer: [%" PRId64 " %" PRId64 sTrace("vgId:%d, already committed. index: %" PRId64 ", term: %" PRId64 ". log buffer: [%" PRId64 " %" PRId64
@ -306,10 +308,9 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
} }
// check current in buffer // check current in buffer
SSyncRaftEntry* pExist = pBuf->entries[index % pBuf->size].pItem; pExist = syncLogBufferGetOneEntry(pBuf, pNode, index, &inBuf);
if (pExist != NULL) { if (pExist != NULL) {
ASSERT(pEntry->index == pExist->index); ASSERT(pEntry->index == pExist->index);
if (pEntry->term != pExist->term) { if (pEntry->term != pExist->term) {
(void)syncLogBufferRollback(pBuf, pNode, index); (void)syncLogBufferRollback(pBuf, pNode, index);
} else { } else {
@ -317,14 +318,15 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
" %" PRId64 " %" PRId64 ", %" PRId64 ")", " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex,
pBuf->endIndex); pBuf->endIndex);
SyncTerm existPrevTerm = pBuf->entries[index % pBuf->size].prevLogTerm; SyncTerm existPrevTerm = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index);
ASSERT(pEntry->term == pExist->term && prevTerm == existPrevTerm); ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm));
ret = 0; ret = 0;
goto _out; goto _out;
} }
} }
// update // update
ASSERT(pBuf->entries[index % pBuf->size].pItem == NULL);
SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm}; SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = prevIndex, .prevLogTerm = prevTerm};
pEntry = NULL; pEntry = NULL;
pBuf->entries[index % pBuf->size] = tmp; pBuf->entries[index % pBuf->size] = tmp;
@ -337,6 +339,10 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt
_out: _out:
syncEntryDestroy(pEntry); syncEntryDestroy(pEntry);
if (!inBuf) {
syncEntryDestroy(pExist);
pExist = NULL;
}
syncLogBufferValidate(pBuf); syncLogBufferValidate(pBuf);
taosThreadMutexUnlock(&pBuf->mutex); taosThreadMutexUnlock(&pBuf->mutex);
return ret; return ret;
@ -1008,6 +1014,16 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex
lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore);
ASSERT(toIndex == lastVer + 1); ASSERT(toIndex == lastVer + 1);
// refill buffer on need
if (toIndex <= pBuf->startIndex) {
int32_t ret = syncLogBufferInitWithoutLock(pBuf, pNode);
if (ret < 0) {
sError("vgId:%d, failed to refill sync log buffer since %s", pNode->vgId, terrstr());
return -1;
}
}
ASSERT(pBuf->endIndex == toIndex);
syncLogBufferValidate(pBuf); syncLogBufferValidate(pBuf);
return 0; return 0;
} }

28
source/libs/tdb/src/db/tdbPCache.c
View File

@ -192,6 +192,28 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) {
return pPage; return pPage;
} }
void tdbPCacheMarkFree(SPCache *pCache, SPage *pPage) {
tdbPCacheLock(pCache);
tdbPCacheRemovePageFromHash(pCache, pPage);
pPage->isFree = 1;
tdbPCacheUnlock(pCache);
}
static void tdbPCacheFreePage(SPCache *pCache, SPage *pPage) {
if (pPage->id < pCache->nPages) {
pPage->pFreeNext = pCache->pFree;
pCache->pFree = pPage;
pPage->isFree = 0;
++pCache->nFree;
tdbTrace("pcache/free page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
} else {
tdbTrace("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
tdbPCacheRemovePageFromHash(pCache, pPage);
tdbPageDestroy(pPage, tdbDefaultFree, NULL);
}
}
void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
i32 nRef; i32 nRef;
@ -209,7 +231,11 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) {
// nRef = tdbGetPageRef(pPage); // nRef = tdbGetPageRef(pPage);
// if (nRef == 0) { // if (nRef == 0) {
if (pPage->isLocal) { if (pPage->isLocal) {
tdbPCacheUnpinPage(pCache, pPage); if (!pPage->isFree) {
tdbPCacheUnpinPage(pCache, pPage);
} else {
tdbPCacheFreePage(pCache, pPage);
}
} else { } else {
if (TDB_TXN_IS_WRITE(pTxn)) { if (TDB_TXN_IS_WRITE(pTxn)) {
// remove from hash // remove from hash

3
source/libs/tdb/src/db/tdbPage.c
View File

@ -111,6 +111,9 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) {
tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize);
pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pPageHdr = pPage->pData + szAmHdr;
if (TDB_PAGE_NCELLS(pPage) == 0) {
return tdbPageZero(pPage, szAmHdr, xCellSize);
}
pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage);
pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage);
pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage); pPage->pFreeEnd = pPage->pData + TDB_PAGE_CCELLS(pPage);

21
source/libs/tdb/src/db/tdbPager.c
View File

@ -466,11 +466,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
return -1; return -1;
} }
if (tdbOsLSeek(jfd, 0L, SEEK_SET) < 0) {
tdbError("failed to lseek jfd due to %s. file:%s, offset:0", strerror(errno), pPager->dbFileName);
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize); u8 *pageBuf = tdbOsCalloc(1, pPager->pageSize);
if (pageBuf == NULL) { if (pageBuf == NULL) {
return -1; return -1;
} }
tdbDebug("tdb/abort: pager:%p,", pPager);
for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) {
// read pgno & the page from journal // read pgno & the page from journal
SPgno pgno; SPgno pgno;
@ -481,6 +489,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
return -1; return -1;
} }
tdbTrace("tdb/abort: pgno:%d,", pgno);
ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); ret = tdbOsRead(jfd, pageBuf, pPager->pageSize);
if (ret < 0) { if (ret < 0) {
tdbOsFree(pageBuf); tdbOsFree(pageBuf);
@ -524,6 +534,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage)));
tdbPCacheMarkFree(pPager->pCache, pPage);
tdbPCacheRelease(pPager->pCache, pPage, pTxn); tdbPCacheRelease(pPager->pCache, pPage, pTxn);
} }
@ -577,12 +588,12 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
return -1; return -1;
} }
tdbTrace("tdb/flush:%p, %d/%d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize, maxPgno); tdbTrace("tdb/flush:%p, pgno:%d, %d/%d/%d", pPager, pgno, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
pPager->dbOrigSize = maxPgno; pPager->dbOrigSize = maxPgno;
pPage->isDirty = 0; pPage->isDirty = 0;
tdbTrace("pager/flush drop page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); tdbTrace("pager/flush drop page: %p, pgno:%d, from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt);
tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
tdbPCacheRelease(pPager->pCache, pPage, pTxn); tdbPCacheRelease(pPager->pCache, pPage, pTxn);
@ -829,7 +840,7 @@ static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) {
return 0; return 0;
} }
static int tdbPagerRestore(SPager *pPager, SBTree *pBt, const char *jFileName) { static int tdbPagerRestore(SPager *pPager, const char *jFileName) {
int ret = 0; int ret = 0;
SPgno journalSize = 0; SPgno journalSize = 0;
u8 *pageBuf = NULL; u8 *pageBuf = NULL;
@ -907,7 +918,7 @@ static int tdbPagerRestore(SPager *pPager, SBTree *pBt, const char *jFileName) {
return 0; return 0;
} }
int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) { int tdbPagerRestoreJournals(SPager *pPager) {
tdbDirEntryPtr pDirEntry; tdbDirEntryPtr pDirEntry;
tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName);
if (pDir == NULL) { if (pDir == NULL) {
@ -918,7 +929,7 @@ int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) {
while ((pDirEntry = tdbReadDir(pDir)) != NULL) { while ((pDirEntry = tdbReadDir(pDir)) != NULL) {
char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry));
if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) {
if (tdbPagerRestore(pPager, pBt, name) < 0) { if (tdbPagerRestore(pPager, name) < 0) {
tdbCloseDir(&pDir); tdbCloseDir(&pDir);
tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name);

20
source/libs/tdb/src/db/tdbTable.c
View File

@ -107,6 +107,16 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
ASSERT(pPager != NULL); ASSERT(pPager != NULL);
if (rollback) {
tdbPagerRollback(pPager);
} else {
ret = tdbPagerRestoreJournals(pPager);
if (ret < 0) {
tdbOsFree(pTb);
return -1;
}
}
// pTb->pBt // pTb->pBt
ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt));
if (ret < 0) { if (ret < 0) {
@ -114,16 +124,6 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF
return -1; return -1;
} }
if (rollback) {
tdbPagerRollback(pPager);
} else {
ret = tdbPagerRestoreJournals(pPager, pTb->pBt);
if (ret < 0) {
tdbOsFree(pTb);
return -1;
}
}
*ppTb = pTb; *ppTb = pTb;
return 0; return 0;
} }

4
source/libs/tdb/src/inc/tdbInt.h
View File

@ -197,7 +197,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP
TXN *pTxn); TXN *pTxn);
void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt); int tdbPagerRestoreJournals(SPager *pPager);
int tdbPagerRollback(SPager *pPager); int tdbPagerRollback(SPager *pPager);
// tdbPCache.c ==================================== // tdbPCache.c ====================================
@ -205,6 +205,7 @@ int tdbPagerRollback(SPager *pPager);
u8 isAnchor; \ u8 isAnchor; \
u8 isLocal; \ u8 isLocal; \
u8 isDirty; \ u8 isDirty; \
u8 isFree; \
volatile i32 nRef; \ volatile i32 nRef; \
i32 id; \ i32 id; \
SPage *pFreeNext; \ SPage *pFreeNext; \
@ -222,6 +223,7 @@ int tdbPCacheClose(SPCache *pCache);
int tdbPCacheAlter(SPCache *pCache, int32_t nPage); int tdbPCacheAlter(SPCache *pCache, int32_t nPage);
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn); SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn);
void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn); void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn);
void tdbPCacheMarkFree(SPCache *pCache, SPage *pPage);
int tdbPCacheGetPageSize(SPCache *pCache); int tdbPCacheGetPageSize(SPCache *pCache);
// tdbPage.c ==================================== // tdbPage.c ====================================

4
source/libs/transport/src/transCli.c
View File

@ -599,6 +599,10 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
exh->pThrd = conn->hostThrd; exh->pThrd = conn->hostThrd;
exh->refId = transAddExHandle(transGetRefMgt(), exh); exh->refId = transAddExHandle(transGetRefMgt(), exh);
conn->refId = exh->refId; conn->refId = exh->refId;
if (conn->refId == -1) {
taosMemoryFree(exh);
}
return 0; return 0;
} }

31
source/libs/wal/src/walMeta.c
View File

@ -25,7 +25,7 @@ bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) {
} }
bool FORCE_INLINE walIsEmpty(SWal* pWal) { bool FORCE_INLINE walIsEmpty(SWal* pWal) {
return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1)
} }
int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
@ -49,7 +49,6 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
ASSERT(fileIdx >= 0 && fileIdx < sz);
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -101,7 +100,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
offsetBackward = offset; offsetBackward = offset;
} }
ASSERT(offset <= end);
readSize = end - offset; readSize = end - offset;
capacity = readSize + sizeof(magic); capacity = readSize + sizeof(magic);
@ -257,7 +255,6 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i);
while (j < metaFileNum) { while (j < metaFileNum) {
SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j);
ASSERT(pMetaInfo != NULL);
if (pMetaInfo->firstVer < pLogInfo->firstVer) { if (pMetaInfo->firstVer < pLogInfo->firstVer) {
j++; j++;
} else if (pMetaInfo->firstVer == pLogInfo->firstVer) { } else if (pMetaInfo->firstVer == pLogInfo->firstVer) {
@ -385,7 +382,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
taosArrayDestroy(actualLog); taosArrayDestroy(actualLog);
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
ASSERT(sz == actualFileNum);
// scan and determine the lastVer // scan and determine the lastVer
int32_t fileIdx = sz; int32_t fileIdx = sz;
@ -403,8 +399,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
return -1; return -1;
} }
ASSERT(pFileInfo->firstVer >= 0);
if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) {
totSize += pFileInfo->fileSize; totSize += pFileInfo->fileSize;
continue; continue;
@ -417,7 +411,6 @@ int walCheckAndRepairMeta(SWal* pWal) {
wError("failed to scan wal last ver since %s", terrstr()); wError("failed to scan wal last ver since %s", terrstr());
return -1; return -1;
} }
ASSERT(pFileInfo->fileSize == 0);
// remove the empty wal log, and its idx // remove the empty wal log, and its idx
wInfo("vgId:%d, wal remove empty file %s", pWal->cfg.vgId, fnameStr); wInfo("vgId:%d, wal remove empty file %s", pWal->cfg.vgId, fnameStr);
taosRemoveFile(fnameStr); taosRemoveFile(fnameStr);
@ -477,8 +470,7 @@ int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
} }
int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
ASSERT(fileIdx >= 0 && fileIdx < sz);
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
@ -492,7 +484,6 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
return -1; return -1;
} }
ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer);
if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) {
return 0; return 0;
} }
@ -556,7 +547,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
} }
offset += sizeof(SWalIdxEntry); offset += sizeof(SWalIdxEntry);
ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); /*A(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry));*/
// ftruncate idx file // ftruncate idx file
if (offset < fileSize) { if (offset < fileSize) {
@ -577,7 +568,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
} }
while (idxEntry.ver < pFileInfo->lastVer) { while (idxEntry.ver < pFileInfo->lastVer) {
ASSERT(idxEntry.ver == ckHead.head.version); /*A(idxEntry.ver == ckHead.head.version);*/
idxEntry.ver += 1; idxEntry.ver += 1;
idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen;
@ -649,8 +640,7 @@ int walRollFileInfo(SWal* pWal) {
} }
char* walMetaSerialize(SWal* pWal) { char* walMetaSerialize(SWal* pWal) {
char buf[30]; char buf[30];
ASSERT(pWal->fileInfoSet);
int sz = taosArrayGetSize(pWal->fileInfoSet); int sz = taosArrayGetSize(pWal->fileInfoSet);
cJSON* pRoot = cJSON_CreateObject(); cJSON* pRoot = cJSON_CreateObject();
cJSON* pMeta = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject();
@ -707,7 +697,7 @@ char* walMetaSerialize(SWal* pWal) {
} }
int walMetaDeserialize(SWal* pWal, const char* bytes) { int walMetaDeserialize(SWal* pWal, const char* bytes) {
ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); /*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/
cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField;
pRoot = cJSON_Parse(bytes); pRoot = cJSON_Parse(bytes);
if (!pRoot) goto _err; if (!pRoot) goto _err;
@ -823,7 +813,9 @@ int walSaveMeta(SWal* pWal) {
// flush to a tmpfile // flush to a tmpfile
n = walBuildTmpMetaName(pWal, tmpFnameStr); n = walBuildTmpMetaName(pWal, tmpFnameStr);
ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); if (n >= sizeof(tmpFnameStr)) {
return -1;
}
TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pMetaFile == NULL) { if (pMetaFile == NULL) {
@ -854,7 +846,9 @@ int walSaveMeta(SWal* pWal) {
// rename it // rename it
n = walBuildMetaName(pWal, metaVer + 1, fnameStr); n = walBuildMetaName(pWal, metaVer + 1, fnameStr);
ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); if (n >= sizeof(fnameStr)) {
goto _err;
}
if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { if (taosRenameFile(tmpFnameStr, fnameStr) < 0) {
wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr);
@ -877,7 +871,6 @@ _err:
} }
int walLoadMeta(SWal* pWal) { int walLoadMeta(SWal* pWal) {
ASSERT(pWal->fileInfoSet->size == 0);
// find existing meta file // find existing meta file
int metaVer = walFindCurMetaVer(pWal); int metaVer = walFindCurMetaVer(pWal);
if (metaVer == -1) { if (metaVer == -1) {

19
source/libs/wal/src/walRead.c
View File

@ -97,7 +97,6 @@ int32_t walNextValidMsg(SWalReader *pReader) {
return -1; return -1;
} }
fetchVer++; fetchVer++;
ASSERT(fetchVer == pReader->curVersion);
} }
} }
pReader->curStopped = 1; pReader->curStopped = 1;
@ -132,7 +131,6 @@ static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int
return -1; return -1;
} }
ASSERT(entry.ver == ver);
ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (ret < 0) { if (ret < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -241,7 +239,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
if (pRead->curInvalid || pRead->curVersion != fetchVer) { if (pRead->curInvalid || pRead->curVersion != fetchVer) {
if (walReadSeekVer(pRead, fetchVer) < 0) { if (walReadSeekVer(pRead, fetchVer) < 0) {
ASSERT(0);
pRead->curVersion = fetchVer; pRead->curVersion = fetchVer;
pRead->curInvalid = 1; pRead->curInvalid = 1;
return -1; return -1;
@ -262,7 +259,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) {
} else { } else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
ASSERT(0);
pRead->curInvalid = 1; pRead->curInvalid = 1;
return -1; return -1;
} }
@ -300,7 +296,6 @@ static int32_t walFetchBodyNew(SWalReader *pReader) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
pReader->curInvalid = 1; pReader->curInvalid = 1;
ASSERT(0);
return -1; return -1;
} }
@ -309,7 +304,6 @@ static int32_t walFetchBodyNew(SWalReader *pReader) {
pReader->pHead->head.version, ver); pReader->pHead->head.version, ver);
pReader->curInvalid = 1; pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1; return -1;
} }
@ -317,7 +311,6 @@ static int32_t walFetchBodyNew(SWalReader *pReader) {
wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver);
pReader->curInvalid = 1; pReader->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
return -1; return -1;
} }
@ -329,14 +322,10 @@ static int32_t walFetchBodyNew(SWalReader *pReader) {
static int32_t walSkipFetchBodyNew(SWalReader *pRead) { static int32_t walSkipFetchBodyNew(SWalReader *pRead) {
int64_t code; int64_t code;
ASSERT(pRead->curVersion == pRead->pHead->head.version);
ASSERT(pRead->curInvalid == 0);
code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
pRead->curInvalid = 1; pRead->curInvalid = 1;
ASSERT(0);
return -1; return -1;
} }
@ -385,7 +374,6 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
} else { } else {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
ASSERT(0);
pRead->curInvalid = 1; pRead->curInvalid = 1;
return -1; return -1;
} }
@ -411,9 +399,6 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
pRead->pWal->cfg.vgId, pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->cfg.vgId, pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer); pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer);
ASSERT(pRead->curVersion == pHead->head.version);
ASSERT(pRead->curInvalid == 0);
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -448,7 +433,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) {
if (pReadHead->bodyLen < 0) { if (pReadHead->bodyLen < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s",
pRead->pWal->cfg.vgId, pReadHead->version, ver, tstrerror(terrno)); pRead->pWal->cfg.vgId, pReadHead->version, ver, tstrerror(terrno));
@ -458,12 +442,10 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
pRead->curInvalid = 1; pRead->curInvalid = 1;
ASSERT(0);
return -1; return -1;
} }
if (pReadHead->version != ver) { if (pReadHead->version != ver) {
ASSERT(0);
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, wError("vgId:%d, wal fetch body error, index:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId,
pReadHead->version, ver); pReadHead->version, ver);
pRead->curInvalid = 1; pRead->curInvalid = 1;
@ -472,7 +454,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
} }
if (walValidBodyCksum(*ppHead) != 0) { if (walValidBodyCksum(*ppHead) != 0) {
ASSERT(0);
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
ver); ver);
pRead->curInvalid = 1; pRead->curInvalid = 1;

5
source/libs/wal/src/walRef.c
View File

@ -61,7 +61,7 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) {
SWalFileInfo tmpInfo; SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver; tmpInfo.firstVer = ver;
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL); /*A(pRet != NULL);*/
pRef->refFile = pRet->firstVer; pRef->refFile = pRet->firstVer;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
@ -80,6 +80,7 @@ void walUnrefVer(SWalRef *pRef) {
SWalRef *walRefCommittedVer(SWal *pWal) { SWalRef *walRefCommittedVer(SWal *pWal) {
SWalRef *pRef = walOpenRef(pWal); SWalRef *pRef = walOpenRef(pWal);
if (pRef == NULL) { if (pRef == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
taosThreadMutexLock(&pWal->mutex); taosThreadMutexLock(&pWal->mutex);
@ -91,7 +92,7 @@ SWalRef *walRefCommittedVer(SWal *pWal) {
SWalFileInfo tmpInfo; SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver; tmpInfo.firstVer = ver;
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL); /*A(pRet != NULL);*/
pRef->refFile = pRet->firstVer; pRef->refFile = pRet->firstVer;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);

7
source/libs/wal/src/walSeek.c
View File

@ -40,7 +40,6 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
ASSERT(entry.ver == ver);
code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
@ -53,8 +52,7 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) {
int walInitWriteFile(SWal* pWal) { int walInitWriteFile(SWal* pWal) {
TdFilePtr pIdxTFile, pLogTFile; TdFilePtr pIdxTFile, pLogTFile;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer;
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr); walBuildIdxName(pWal, fileFirstVer, fnameStr);
@ -109,9 +107,8 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) {
tmpInfo.firstVer = ver; tmpInfo.firstVer = ver;
// bsearch in fileSet // bsearch in fileSet
int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(idx != -1); /*A(idx != -1);*/
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx);
/*ASSERT(pFileInfo != NULL);*/
int64_t fileFirstVer = pFileInfo->firstVer; int64_t fileFirstVer = pFileInfo->firstVer;
walBuildIdxName(pWal, fileFirstVer, fnameStr); walBuildIdxName(pWal, fileFirstVer, fnameStr);

38
source/libs/wal/src/walWrite.c
View File

@ -87,12 +87,10 @@ int32_t walApplyVer(SWal *pWal, int64_t ver) {
} }
int32_t walCommit(SWal *pWal, int64_t ver) { int32_t walCommit(SWal *pWal, int64_t ver) {
ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer);
ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer);
if (ver < pWal->vers.commitVer) { if (ver < pWal->vers.commitVer) {
return 0; return 0;
} }
if (ver > pWal->vers.lastVer) { if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) {
terrno = TSDB_CODE_WAL_INVALID_VER; terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; return -1;
} }
@ -138,25 +136,21 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); TdFilePtr pIdxFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
if (pIdxFile == NULL) { if (pIdxFile == NULL) {
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
int64_t idxOff = walGetVerIdxOffset(pWal, ver); int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET); code = taosLSeekFile(pIdxFile, idxOff, SEEK_SET);
if (code < 0) { if (code < 0) {
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
// read idx file and get log file pos // read idx file and get log file pos
SWalIdxEntry entry; SWalIdxEntry entry;
if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { if (taosReadFile(pIdxFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
ASSERT(entry.ver == ver);
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
@ -176,24 +170,19 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
} }
// validate offset // validate offset
SWalCkHead head; SWalCkHead head;
ASSERT(taosValidFile(pLogFile)); int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead));
int64_t size = taosReadFile(pLogFile, &head, sizeof(SWalCkHead));
if (size != sizeof(SWalCkHead)) { if (size != sizeof(SWalCkHead)) {
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
code = walValidHeadCksum(&head); code = walValidHeadCksum(&head);
ASSERT(code == 0);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
ASSERT(0);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
if (head.head.version != ver) { if (head.head.version != ver) {
ASSERT(0);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
@ -202,22 +191,22 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
// truncate old files // truncate old files
code = taosFtruncateFile(pLogFile, entry.offset); code = taosFtruncateFile(pLogFile, entry.offset);
if (code < 0) { if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
code = taosFtruncateFile(pIdxFile, idxOff); code = taosFtruncateFile(pIdxFile, idxOff);
if (code < 0) { if (code < 0) {
ASSERT(0);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
} }
pWal->vers.lastVer = ver - 1; pWal->vers.lastVer = ver - 1;
#if 0
if (pWal->vers.lastVer < pWal->vers.firstVer) { if (pWal->vers.lastVer < pWal->vers.firstVer) {
ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1); A(pWal->vers.lastVer == pWal->vers.firstVer - 1);
} }
#endif
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
taosCloseFile(&pIdxFile); taosCloseFile(&pIdxFile);
@ -386,7 +375,8 @@ int32_t walEndSnapshot(SWal *pWal) {
walBuildIdxName(pWal, pInfo->firstVer, fnameStr); walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) {
ASSERT(0); wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno));
goto END;
} }
} }
taosArrayClear(pWal->toDeleteFiles); taosArrayClear(pWal->toDeleteFiles);
@ -441,7 +431,6 @@ int32_t walRollImpl(SWal *pWal) {
pWal->pIdxFile = pIdxFile; pWal->pIdxFile = pIdxFile;
pWal->pLogFile = pLogFile; pWal->pLogFile = pLogFile;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
ASSERT(pWal->writeCur >= 0);
pWal->lastRollSeq = walGetSeq(); pWal->lastRollSeq = walGetSeq();
@ -458,8 +447,7 @@ END:
static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
SWalIdxEntry entry = {.ver = ver, .offset = offset}; SWalIdxEntry entry = {.ver = ver, .offset = offset};
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
ASSERT(pFileInfo != NULL);
ASSERT(pFileInfo->firstVer >= 0);
int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry);
wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset,
idxOffset); idxOffset);
@ -476,7 +464,6 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
if (endOffset < 0) { if (endOffset < 0) {
wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver); wFatal("vgId:%d, failed to seek end of idxfile due to %s. ver:%" PRId64 "", pWal->cfg.vgId, strerror(errno), ver);
} }
ASSERT(endOffset == idxOffset + sizeof(SWalIdxEntry) && "Offset of idx entries misaligned");
return 0; return 0;
} }
@ -486,9 +473,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
int64_t offset = walGetCurFileOffset(pWal); int64_t offset = walGetCurFileOffset(pWal);
SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal);
ASSERT(pFileInfo != NULL);
ASSERT(pFileInfo->firstVer != -1);
pWal->writeHead.head.version = index; pWal->writeHead.head.version = index;
pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.bodyLen = bodyLen;
pWal->writeHead.head.msgType = msgType; pWal->writeHead.head.msgType = msgType;
@ -525,7 +510,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy
// set status // set status
if (pWal->vers.firstVer == -1) { if (pWal->vers.firstVer == -1) {
ASSERT(index == 0);
pWal->vers.firstVer = 0; pWal->vers.firstVer = 0;
} }
pWal->vers.lastVer = index; pWal->vers.lastVer = index;
@ -541,7 +525,6 @@ END:
wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId, wFatal("vgId:%d, failed to ftruncate logfile to offset:%" PRId64 " during recovery due to %s", pWal->cfg.vgId,
offset, strerror(errno)); offset, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
ASSERT(0 && "failed to recover from error");
} }
int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry);
@ -549,7 +532,6 @@ END:
wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId, wFatal("vgId:%d, failed to ftruncate idxfile to offset:%" PRId64 "during recovery due to %s", pWal->cfg.vgId,
idxOffset, strerror(errno)); idxOffset, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
ASSERT(0 && "failed to recover from error");
} }
return -1; return -1;
} }
@ -576,8 +558,6 @@ int64_t walAppendLog(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syn
} }
} }
ASSERT(pWal->pLogFile != NULL && pWal->pIdxFile != NULL && pWal->writeCur >= 0);
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;
@ -614,8 +594,6 @@ int32_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SWalSync
} }
} }
ASSERT(pWal->pIdxFile != NULL && pWal->pLogFile != NULL && pWal->writeCur >= 0);
if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) { if (walWriteImpl(pWal, index, msgType, syncMeta, body, bodyLen) < 0) {
taosThreadMutexUnlock(&pWal->mutex); taosThreadMutexUnlock(&pWal->mutex);
return -1; return -1;

2
tests/develop-test/5-taos-tools/taosbenchmark/auto_create_table_json.py
View File

@ -153,6 +153,8 @@ class TDTestCase:
tdSql.query("select count(*) from db.`stb4-2`") tdSql.query("select count(*) from db.`stb4-2`")
tdSql.checkData(0, 0, 160) tdSql.checkData(0, 0, 160)
tAdapter.stop()
def stop(self): def stop(self):
tdSql.close() tdSql.close()
tdLog.success("%s successfully executed" % __file__) tdLog.success("%s successfully executed" % __file__)

1
tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py
View File

@ -321,6 +321,7 @@ class TDTestCase:
tdSql.query("select count(*) from db.stb where t13 like 'b1%' or t13 like 'b2%'") tdSql.query("select count(*) from db.stb where t13 like 'b1%' or t13 like 'b2%'")
tdSql.checkData(0, 0, 160) tdSql.checkData(0, 0, 160)
tAdapter.stop()
def stop(self): def stop(self):
tdSql.close() tdSql.close()

2
tests/develop-test/5-taos-tools/taosbenchmark/query_json.py
View File

@ -116,7 +116,7 @@ class TDTestCase:
assert times == 1, "result is %s != expect: 1" % times assert times == 1, "result is %s != expect: 1" % times
tAdapter.stop()
def stop(self): def stop(self):

11
tests/parallel_test/cases.task
View File

@ -481,7 +481,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py -N 3 -n 3 -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py
@ -648,6 +648,9 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3
@ -658,6 +661,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
@ -1018,6 +1022,11 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -R
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
#develop test #develop test

2
tests/pytest/util/cluster.py
View File

@ -53,7 +53,7 @@ class ConfigureyCluster:
# configure dnoe of independent mnodes # configure dnoe of independent mnodes
if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True : if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True :
dnode.addExtraCfg("supportVnodes", 0) dnode.addExtraCfg("supportVnodes", 1024)
# print(dnode) # print(dnode)
self.dnodes.append(dnode) self.dnodes.append(dnode)
return self.dnodes return self.dnodes

Some files were not shown because too many files have changed in this diff Show More