xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feat/TS-2158

This commit is contained in:
Ganlin Zhao 2022-12-19 09:23:32 +08:00
parent bb6e23ed06 a678d6326d
commit 09061bcc57
117 changed files with 5175 additions and 5925 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
cmake/cmake.define
View File

@ -123,21 +123,37 @@ ELSE ()
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
ENDIF () ENDIF ()
IF (TD_INTEL_64 OR TD_INTEL_32) INCLUDE(CheckCCompilerFlag)
ADD_DEFINITIONS("-msse4.2") IF (("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") OR ("${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang"))
IF("${FMA_SUPPORT}" MATCHES "true") SET(COMPILER_SUPPORT_SSE42 true)
MESSAGE(STATUS "fma function supported") MESSAGE(STATUS "Always enable sse4.2 for Clang/AppleClang")
ADD_DEFINITIONS("-mfma") ELSE()
ELSE () CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE42)
MESSAGE(STATUS "fma function NOT supported") ENDIF()
ENDIF()
IF("${SIMD_SUPPORT}" MATCHES "true") CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA)
ADD_DEFINITIONS("-mavx -mavx2") CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX)
MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2)
ELSE()
MESSAGE(STATUS "SIMD instruction (AVX/AVX2)is NOT ACTIVATED") IF (COMPILER_SUPPORT_SSE42)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
ENDIF()
IF (COMPILER_SUPPORT_FMA)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma")
ENDIF()
IF ("${SIMD_SUPPORT}" MATCHES "true")
IF (COMPILER_SUPPORT_AVX)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
ENDIF() ENDIF()
ENDIF () IF (COMPILER_SUPPORT_AVX2)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
ENDIF()
MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED")
ENDIF()
ENDIF () ENDIF ()

2
cmake/cmake.platform
View File

@ -147,7 +147,7 @@ ELSE ()
ENDIF () ENDIF ()
ENDIF () ENDIF ()
MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR}) MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR})
MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})") MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})")
MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})") MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})")

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG 121c8a3 GIT_TAG b20c9d1
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

1
docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx
View File

@ -47,7 +47,6 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http
:::note :::note
- In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type.
- Only data in array format is accepted and so an array must be used even if there is only one row.
- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. - The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
::: :::

45
docs/en/12-taos-sql/14-stream.md
View File

@ -10,7 +10,7 @@ Because stream processing is built in to TDengine, you are no longer reliant on
## Create a Stream ## Create a Stream
```sql ```sql
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery
stream_options: { stream_options: {
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time]
WATERMARK time WATERMARK time
@ -30,6 +30,8 @@ subquery: SELECT [DISTINCT] select_list
Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME. Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME.
Subtable Clause defines the naming rules of auto-created subtable, you can see more details in below part: Partitions of Stream.
```sql ```sql
window_clause: { window_clause: {
SESSION(ts_col, tol_val) SESSION(ts_col, tol_val)
@ -47,6 +49,47 @@ CREATE STREAM avg_vol_s INTO avg_vol AS
SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s);
``` ```
## Partitions of Stream
A Stream can process data in multiple partitions. Partition rules can be defined by PARTITION BY clause in stream processing. Each partition will have different timelines and windows, and will be processed separately and be written into different subtables of target supertable.
If a stream is created without PARTITION BY clause, all data will be written into one subtable.
If a stream is created with PARTITION BY clause without SUBTABLE clause, each partition will be given a random name.
If a stream is created with PARTITION BY clause and SUBTABLE clause, the name of each partition will be calculated according to SUBTABLE clause. For example:
```sql
CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
```
IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name. Other expressions are also allowed in SUBTABLE clause, but the output type must be varchar.
If the output length exceeds the limitation of TDengine(192), the name will be truncated. If the generated name is occupied by some other table, the creation and writing of the new subtable will be failed.
## Filling history data
Normally a stream does not process data already or being written into source table when it's being creating. But adding FILL_HISTORY 1 as a stream option when creating the stream will allow it to process data written before and while creating the stream. For example:
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s)
```
Combining fill_history option and where clause, stream can processing data of specific time range. For example, only process data after a past time. (In this case, 2020-01-30)
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s)
```
As another example, only processing data starting from some past time, and ending at some future time.
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' and ts < '2023-01-01' interval(10s)
```
If some streams are totally outdated, and you do not want it to monitor or process anymore, those streams can be manually dropped and output data will be still kept.
## Delete a Stream ## Delete a Stream
```sql ```sql

2
docs/en/20-third-party/11-kafka.md vendored
View File

@ -76,7 +76,7 @@ Development: false
### Install from source code ### Install from source code
``` ```
git clone https://github.com:taosdata/kafka-connect-tdengine.git git clone https://github.com/taosdata/kafka-connect-tdengine.git
cd kafka-connect-tdengine cd kafka-connect-tdengine
mvn clean package mvn clean package
unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip

2
docs/examples/python/connection_usage_native_reference.py
View File

@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test")
# change database. same as execute "USE db" # change database. same as execute "USE db"
conn.select_db("test") conn.select_db("test")
conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m 24.4)") affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)")
print("affected_row", affected_row) print("affected_row", affected_row)
# output: # output:
# affected_row 3 # affected_row 3

1
docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx
View File

@ -47,7 +47,6 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据
:::note :::note
- 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。
- TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。
- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 - 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
::: :::

43
docs/zh/12-taos-sql/14-stream.md
View File

@ -8,7 +8,7 @@ description: 流式计算的相关 SQL 的详细语法
## 创建流式计算 ## 创建流式计算
```sql ```sql
CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery
stream_options: { stream_options: {
TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time]
WATERMARK time WATERMARK time
@ -28,6 +28,9 @@ subquery: SELECT select_list
支持会话窗口、状态窗口与滑动窗口，其中，会话窗口与状态窗口搭配超级表时必须与partition by tbname一起使用 支持会话窗口、状态窗口与滑动窗口，其中，会话窗口与状态窗口搭配超级表时必须与partition by tbname一起使用
subtable 子句定义了流式计算中创建的子表的命名规则，详见 流式计算的 partition 部分。
```sql ```sql
window_clause: { window_clause: {
SESSION(ts_col, tol_val) SESSION(ts_col, tol_val)
@ -49,11 +52,43 @@ SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(
## 流式计算的 partition ## 流式计算的 partition
可以使用 PARTITION BY TBNAME 或 PARTITION BY tag，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。 可以使用 PARTITION BY TBNAME，tag，普通列或者表达式，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。
不带 PARTITION BY 选项时，所有的数据将写入到一张子表。 不带 PARTITION BY 子句时，所有的数据将写入到一张子表。
流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。 在创建流时不使用 SUBTABLE 子句时，流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。
若创建流的语句中包含 SUBTABLE 子句，用户可以为每个 partition 对应的子表生成自定义的表名，例如：
```sql
CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m);
```
PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名。
注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。
## 流式计算读取历史数据
正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。例如：
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s)
```
结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020年1月30日）之后的数据
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s)
```
再如，仅处理某时间段内的数据，结束时间可以是未来时间
```sql
create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' and ts < '2023-01-01' interval(10s)
```
如果该流任务已经彻底过期，并且您不再想让它检测或处理数据，您可以手动删除它，被计算出的数据仍会被保留。
## 删除流式计算 ## 删除流式计算

2
docs/zh/20-third-party/11-kafka.md vendored
View File

@ -79,7 +79,7 @@ Development: false
### 从源码安装 ### 从源码安装
``` ```
git clone https://github.com:taosdata/kafka-connect-tdengine.git git clone https://github.com/taosdata/kafka-connect-tdengine.git
cd kafka-connect-tdengine cd kafka-connect-tdengine
mvn clean package mvn clean package
unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip

67
examples/c/CMakeLists.txt
View File

@ -3,19 +3,13 @@ PROJECT(TDengine)
IF (TD_LINUX) IF (TD_LINUX)
INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc)
AUX_SOURCE_DIRECTORY(. SRC) AUX_SOURCE_DIRECTORY(. SRC)
# ADD_EXECUTABLE(demo apitest.c)
#TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
#ADD_EXECUTABLE(sml schemaless.c)
#TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread )
#ADD_EXECUTABLE(subscribe subscribe.c)
#TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread )
#ADD_EXECUTABLE(epoll epoll.c)
#TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
add_executable(tmq "") add_executable(tmq "")
add_executable(stream_demo "") add_executable(stream_demo "")
add_executable(demoapi "") add_executable(schemaless "")
add_executable(api_reqid "") add_executable(prepare "")
add_executable(demo "")
add_executable(asyncdemo "")
target_sources(tmq target_sources(tmq
PRIVATE PRIVATE
@ -27,16 +21,25 @@ IF (TD_LINUX)
"stream_demo.c" "stream_demo.c"
) )
target_sources(demoapi target_sources(schemaless
PRIVATE PRIVATE
"demoapi.c" "schemaless.c"
) )
target_sources(api_reqid target_sources(prepare
PRIVATE PRIVATE
"api_with_reqid_test.c" "prepare.c"
) )
target_sources(demo
PRIVATE
"demo.c"
)
target_sources(asyncdemo
PRIVATE
"asyncdemo.c"
)
target_link_libraries(tmq target_link_libraries(tmq
taos_static taos_static
@ -46,46 +49,30 @@ IF (TD_LINUX)
taos_static taos_static
) )
target_link_libraries(demoapi target_link_libraries(schemaless
taos_static taos_static
) )
target_link_libraries(api_reqid target_link_libraries(prepare
taos_static taos_static
) )
target_link_libraries(demo
target_include_directories(tmq taos_static
PUBLIC "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_include_directories(stream_demo target_link_libraries(asyncdemo
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" taos_static
)
target_include_directories(demoapi
PUBLIC "${TD_SOURCE_DIR}/include/client"
PUBLIC "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_include_directories(api_reqid
PUBLIC "${TD_SOURCE_DIR}/include/client"
PUBLIC "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq)
SET_TARGET_PROPERTIES(stream_demo PROPERTIES OUTPUT_NAME stream_demo) SET_TARGET_PROPERTIES(stream_demo PROPERTIES OUTPUT_NAME stream_demo)
SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi) SET_TARGET_PROPERTIES(schemaless PROPERTIES OUTPUT_NAME schemaless)
SET_TARGET_PROPERTIES(api_reqid PROPERTIES OUTPUT_NAME api_reqid) SET_TARGET_PROPERTIES(prepare PROPERTIES OUTPUT_NAME prepare)
SET_TARGET_PROPERTIES(demo PROPERTIES OUTPUT_NAME demo)
SET_TARGET_PROPERTIES(asyncdemo PROPERTIES OUTPUT_NAME asyncdemo)
ENDIF () ENDIF ()
IF (TD_DARWIN) IF (TD_DARWIN)
INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc)
AUX_SOURCE_DIRECTORY(. SRC) AUX_SOURCE_DIRECTORY(. SRC)
#ADD_EXECUTABLE(demo demo.c)
#TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)
#ADD_EXECUTABLE(epoll epoll.c)
#TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
ENDIF () ENDIF ()

1948
examples/c/apitest.c
View File

File diff suppressed because it is too large Load Diff

8
examples/c/asyncdemo.c
View File

@ -23,8 +23,8 @@
#include <sys/time.h> #include <sys/time.h>
#include <unistd.h> #include <unistd.h>
#include <string.h> #include <string.h>
#include <inttypes.h>
#include "../../../include/client/taos.h" #include "taos.h"
int points = 5; int points = 5;
int numOfTables = 3; int numOfTables = 3;
@ -230,7 +230,7 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code)
if (tablesInsertProcessed >= numOfTables) { if (tablesInsertProcessed >= numOfTables) {
gettimeofday(&systemTime, NULL); gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
printf("%lld mseconds to insert %d data points\n", (et - st) / 1000, points*numOfTables); printf("%" PRId64 " mseconds to insert %d data points\n", (et - st) / 1000, points*numOfTables);
} }
} }
@ -267,7 +267,7 @@ void taos_retrieve_call_back(void *param, TAOS_RES *tres, int numOfRows)
if (tablesSelectProcessed >= numOfTables) { if (tablesSelectProcessed >= numOfTables) {
gettimeofday(&systemTime, NULL); gettimeofday(&systemTime, NULL);
et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; et = systemTime.tv_sec * 1000000 + systemTime.tv_usec;
printf("%lld mseconds to query %d data rows\n", (et - st) / 1000, points * numOfTables); printf("%" PRId64 " mseconds to query %d data rows\n", (et - st) / 1000, points * numOfTables);
} }
taos_free_result(tres); taos_free_result(tres);

2
examples/c/demo.c
View File

@ -20,7 +20,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include "../../../include/client/taos.h" // TAOS header file #include "taos.h" // TAOS header file
static void queryDB(TAOS *taos, char *command) { static void queryDB(TAOS *taos, char *command) {
int i; int i;

304
examples/c/epoll.c
View File

@ -1,304 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// how to use to do a pressure-test upon eok
// tester: cat /dev/urandom | nc -c <ip> <port>
// testee: ./debug/build/bin/epoll -l <port> > /dev/null
// compare against: nc -l <port> > /dev/null
// monitor and compare : glances
#ifdef __APPLE__
#include "osEok.h"
#else // __APPLE__
#include <sys/epoll.h>
#endif // __APPLE__
#include <sys/types.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <arpa/inet.h>
#include <libgen.h>
#include <locale.h>
#include <netdb.h>
#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__)
#define A(statement, fmt, ...) do { \
if (statement) break; \
fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \
basename(__FILE__), __LINE__, __func__, \
#statement, errno, strerror(errno), \
##__VA_ARGS__); \
abort(); \
} while (0)
#define E(fmt, ...) do { \
fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \
basename(__FILE__), __LINE__, __func__, \
errno, strerror(errno), \
##__VA_ARGS__); \
} while (0)
#include "os.h"
typedef struct ep_s ep_t;
struct ep_s {
int ep;
pthread_mutex_t lock;
int sv[2]; // 0 for read, 1 for write;
pthread_t thread;
volatile unsigned int stopping:1;
volatile unsigned int waiting:1;
volatile unsigned int wakenup:1;
};
static int ep_dummy = 0;
static ep_t* ep_create(void);
static void ep_destroy(ep_t *ep);
static void* routine(void* arg);
static int open_listen(unsigned short port);
typedef struct fde_s fde_t;
struct fde_s {
int skt;
void (*on_event)(ep_t *ep, struct epoll_event *events, fde_t *client);
};
static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client);
static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client);
#define usage(arg0, fmt, ...) do { \
if (fmt[0]) { \
fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \
} \
fprintf(stderr, "usage:\n"); \
fprintf(stderr, " %s -l <port> : specify listenning port\n", arg0); \
} while (0)
int main(int argc, char *argv[]) {
char *prg = basename(argv[0]);
if (argc==1) {
usage(prg, "");
return 0;
}
ep_t* ep = ep_create();
A(ep, "failed");
for (int i=1; i<argc; ++i) {
const char *arg = argv[i];
if (0==strcmp(arg, "-l")) {
++i;
if (i>=argc) {
usage(prg, "expecting <port> after -l, but got nothing");
return 1; // confirmed potential leakage
}
arg = argv[i];
int port = atoi(arg);
int skt = open_listen(port);
if (skt==-1) continue;
fde_t *client = (fde_t*)calloc(1, sizeof(*client));
if (!client) {
E("out of memory");
close(skt);
continue;
}
client->skt = skt;
client->on_event = listen_event;
struct epoll_event ev = {0};
ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
ev.data.ptr = client;
A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), "");
continue;
}
usage(prg, "unknown argument: [%s]", arg);
return 1;
}
char *line = NULL;
size_t linecap = 0;
ssize_t linelen;
while ((linelen = getline(&line, &linecap, stdin)) > 0) {
line[strlen(line)-1] = '\0';
if (0==strcmp(line, "exit")) break;
if (0==strcmp(line, "quit")) break;
if (line==strstr(line, "close")) {
int fd = 0;
sscanf(line, "close %d", &fd);
if (fd<=2) {
fprintf(stderr, "fd [%d] invalid\n", fd);
continue;
}
A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), "");
continue;
}
if (strlen(line)==0) continue;
fprintf(stderr, "unknown cmd:[%s]\n", line);
}
ep_destroy(ep);
D("");
return 0;
}
ep_t* ep_create(void) {
ep_t *ep = (ep_t*)calloc(1, sizeof(*ep));
A(ep, "out of memory");
A(-1!=(ep->ep = epoll_create(1)), "");
ep->sv[0] = -1;
ep->sv[1] = -1;
A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), "");
A(0==pthread_mutex_init(&ep->lock, NULL), "");
A(0==pthread_mutex_lock(&ep->lock), "");
struct epoll_event ev = {0};
ev.events = EPOLLIN;
ev.data.ptr = &ep_dummy;
A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), "");
A(0==pthread_create(&ep->thread, NULL, routine, ep), "");
A(0==pthread_mutex_unlock(&ep->lock), "");
return ep;
}
static void ep_destroy(ep_t *ep) {
A(ep, "invalid argument");
ep->stopping = 1;
A(1==send(ep->sv[1], "1", 1, 0), "");
A(0==pthread_join(ep->thread, NULL), "");
A(0==pthread_mutex_destroy(&ep->lock), "");
A(0==close(ep->sv[0]), "");
A(0==close(ep->sv[1]), "");
A(0==close(ep->ep), "");
free(ep);
}
static void* routine(void* arg) {
A(arg, "invalid argument");
ep_t *ep = (ep_t*)arg;
while (!ep->stopping) {
struct epoll_event evs[10];
memset(evs, 0, sizeof(evs));
A(0==pthread_mutex_lock(&ep->lock), "");
A(ep->waiting==0, "internal logic error");
ep->waiting = 1;
A(0==pthread_mutex_unlock(&ep->lock), "");
int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1);
A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r);
A(0==pthread_mutex_lock(&ep->lock), "");
A(ep->waiting==1, "internal logic error");
ep->waiting = 0;
A(0==pthread_mutex_unlock(&ep->lock), "");
for (int i=0; i<r; ++i) {
struct epoll_event *ev = evs + i;
if (ev->data.ptr == &ep_dummy) {
char c = '\0';
A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error");
A(0==pthread_mutex_lock(&ep->lock), "");
ep->wakenup = 0;
A(0==pthread_mutex_unlock(&ep->lock), "");
continue;
}
A(ev->data.ptr, "internal logic error");
fde_t *client = (fde_t*)ev->data.ptr;
client->on_event(ep, ev, client);
continue;
}
}
return NULL;
}
static int open_listen(unsigned short port) {
int r = 0;
int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (skt==-1) {
E("socket() failed");
return -1;
}
do {
struct sockaddr_in si = {0};
si.sin_family = AF_INET;
si.sin_addr.s_addr = inet_addr("0.0.0.0");
si.sin_port = htons(port);
r = bind(skt, (struct sockaddr*)&si, sizeof(si));
if (r) {
E("bind(%u) failed", port);
break;
}
r = listen(skt, 100);
if (r) {
E("listen() failed");
break;
}
memset(&si, 0, sizeof(si));
socklen_t len = sizeof(si);
r = getsockname(skt, (struct sockaddr *)&si, &len);
if (r) {
E("getsockname() failed");
}
A(len==sizeof(si), "internal logic error");
D("listenning at: %d", ntohs(si.sin_port));
return skt;
} while (0);
close(skt);
return -1;
}
static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client) {
A(ev->events & EPOLLIN, "internal logic error");
struct sockaddr_in si = {0};
socklen_t silen = sizeof(si);
int skt = accept(client->skt, (struct sockaddr*)&si, &silen);
A(skt!=-1, "internal logic error");
fde_t *server = (fde_t*)calloc(1, sizeof(*server));
if (!server) {
close(skt);
return;
}
server->skt = skt;
server->on_event = null_event;
struct epoll_event ee = {0};
ee.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
ee.data.ptr = server;
A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ee), "");
}
static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client) {
if (ev->events & EPOLLIN) {
char buf[8192];
int n = recv(client->skt, buf, sizeof(buf), 0);
A(n>=0 && n<=sizeof(buf), "internal logic error:[%d]", n);
A(n==fwrite(buf, 1, n, stdout), "internal logic error");
}
if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) {
A(0==pthread_mutex_lock(&ep->lock), "");
A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), "");
A(0==pthread_mutex_unlock(&ep->lock), "");
close(client->skt);
client->skt = -1;
client->on_event = NULL;
free(client);
}
}

15
examples/c/makefile
View File

@ -7,22 +7,21 @@ LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt
CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \
-Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \
-Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \
-I../../../deps/cJson/inc -I/usr/local/include/cjson
all: $(TARGET) all: $(TARGET)
exe: exe:
gcc $(CFLAGS) ./asyncdemo.c -o $(ROOT)asyncdemo $(LFLAGS) gcc $(CFLAGS) ./asyncdemo.c -o $(ROOT)asyncdemo $(LFLAGS)
gcc $(CFLAGS) ./demo.c -o $(ROOT)demo $(LFLAGS) gcc $(CFLAGS) ./demo.c -o $(ROOT)demo $(LFLAGS)
gcc $(CFLAGS) ./prepare.c -o $(ROOT)prepare $(LFLAGS) gcc $(CFLAGS) ./prepare.c -o $(ROOT)prepare $(LFLAGS)
gcc $(CFLAGS) ./stream.c -o $(ROOT)stream $(LFLAGS) gcc $(CFLAGS) ./stream_demo.c -o $(ROOT)stream_demo $(LFLAGS)
gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) gcc $(CFLAGS) ./tmq.c -o $(ROOT)tmq $(LFLAGS)
gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) gcc $(CFLAGS) ./schemaless.c -o $(ROOT)schemaless $(LFLAGS)
clean: clean:
rm $(ROOT)asyncdemo rm $(ROOT)asyncdemo
rm $(ROOT)demo rm $(ROOT)demo
rm $(ROOT)prepare rm $(ROOT)prepare
rm $(ROOT)batchprepare rm $(ROOT)stream_demo
rm $(ROOT)stream rm $(ROOT)tmq
rm $(ROOT)subscribe rm $(ROOT)schemaless
rm $(ROOT)apitest

49
examples/c/prepare.c
View File

@ -4,7 +4,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include "../../../include/client/taos.h" #include "taos.h"
void taosMsleep(int mseconds); void taosMsleep(int mseconds);
@ -70,70 +70,89 @@ int main(int argc, char *argv[])
char blob[80]; char blob[80];
} v = {0}; } v = {0};
int32_t boolLen = sizeof(int8_t);
int32_t sintLen = sizeof(int16_t);
int32_t intLen = sizeof(int32_t);
int32_t bintLen = sizeof(int64_t);
int32_t floatLen = sizeof(float);
int32_t doubleLen = sizeof(double);
int32_t binLen = sizeof(v.bin);
int32_t ncharLen = 30;
stmt = taos_stmt_init(taos); stmt = taos_stmt_init(taos);
TAOS_BIND params[10]; TAOS_MULTI_BIND params[10];
params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP;
params[0].buffer_length = sizeof(v.ts); params[0].buffer_length = sizeof(v.ts);
params[0].buffer = &v.ts; params[0].buffer = &v.ts;
params[0].length = &params[0].buffer_length; params[0].length = &bintLen;
params[0].is_null = NULL; params[0].is_null = NULL;
params[0].num = 1;
params[1].buffer_type = TSDB_DATA_TYPE_BOOL; params[1].buffer_type = TSDB_DATA_TYPE_BOOL;
params[1].buffer_length = sizeof(v.b); params[1].buffer_length = sizeof(v.b);
params[1].buffer = &v.b; params[1].buffer = &v.b;
params[1].length = &params[1].buffer_length; params[1].length = &boolLen;
params[1].is_null = NULL; params[1].is_null = NULL;
params[1].num = 1;
params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; params[2].buffer_type = TSDB_DATA_TYPE_TINYINT;
params[2].buffer_length = sizeof(v.v1); params[2].buffer_length = sizeof(v.v1);
params[2].buffer = &v.v1; params[2].buffer = &v.v1;
params[2].length = &params[2].buffer_length; params[2].length = &boolLen;
params[2].is_null = NULL; params[2].is_null = NULL;
params[2].num = 1;
params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT;
params[3].buffer_length = sizeof(v.v2); params[3].buffer_length = sizeof(v.v2);
params[3].buffer = &v.v2; params[3].buffer = &v.v2;
params[3].length = &params[3].buffer_length; params[3].length = &sintLen;
params[3].is_null = NULL; params[3].is_null = NULL;
params[3].num = 1;
params[4].buffer_type = TSDB_DATA_TYPE_INT; params[4].buffer_type = TSDB_DATA_TYPE_INT;
params[4].buffer_length = sizeof(v.v4); params[4].buffer_length = sizeof(v.v4);
params[4].buffer = &v.v4; params[4].buffer = &v.v4;
params[4].length = &params[4].buffer_length; params[4].length = &intLen;
params[4].is_null = NULL; params[4].is_null = NULL;
params[4].num = 1;
params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; params[5].buffer_type = TSDB_DATA_TYPE_BIGINT;
params[5].buffer_length = sizeof(v.v8); params[5].buffer_length = sizeof(v.v8);
params[5].buffer = &v.v8; params[5].buffer = &v.v8;
params[5].length = &params[5].buffer_length; params[5].length = &bintLen;
params[5].is_null = NULL; params[5].is_null = NULL;
params[5].num = 1;
params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; params[6].buffer_type = TSDB_DATA_TYPE_FLOAT;
params[6].buffer_length = sizeof(v.f4); params[6].buffer_length = sizeof(v.f4);
params[6].buffer = &v.f4; params[6].buffer = &v.f4;
params[6].length = &params[6].buffer_length; params[6].length = &floatLen;
params[6].is_null = NULL; params[6].is_null = NULL;
params[6].num = 1;
params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE;
params[7].buffer_length = sizeof(v.f8); params[7].buffer_length = sizeof(v.f8);
params[7].buffer = &v.f8; params[7].buffer = &v.f8;
params[7].length = &params[7].buffer_length; params[7].length = &doubleLen;
params[7].is_null = NULL; params[7].is_null = NULL;
params[7].num = 1;
params[8].buffer_type = TSDB_DATA_TYPE_BINARY; params[8].buffer_type = TSDB_DATA_TYPE_BINARY;
params[8].buffer_length = sizeof(v.bin); params[8].buffer_length = sizeof(v.bin);
params[8].buffer = v.bin; params[8].buffer = v.bin;
params[8].length = &params[8].buffer_length; params[8].length = &binLen;
params[8].is_null = NULL; params[8].is_null = NULL;
params[8].num = 1;
strcpy(v.blob, "一二三四五六七八九十"); strcpy(v.blob, "一二三四五六七八九十");
params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; params[9].buffer_type = TSDB_DATA_TYPE_NCHAR;
params[9].buffer_length = strlen(v.blob); params[9].buffer_length = sizeof(v.blob);
params[9].buffer = v.blob; params[9].buffer = v.blob;
params[9].length = &params[9].buffer_length; params[9].length = &ncharLen;
params[9].is_null = NULL; params[9].is_null = NULL;
params[9].num = 1;
int is_null = 1; char is_null = 1;
sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)";
code = taos_stmt_prepare(stmt, sql, 0); code = taos_stmt_prepare(stmt, sql, 0);
@ -153,7 +172,7 @@ int main(int argc, char *argv[])
v.v8 = (int64_t)(i * 8); v.v8 = (int64_t)(i * 8);
v.f4 = (float)(i * 40); v.f4 = (float)(i * 40);
v.f8 = (double)(i * 80); v.f8 = (double)(i * 80);
for (int j = 0; j < sizeof(v.bin) - 1; ++j) { for (int j = 0; j < sizeof(v.bin); ++j) {
v.bin[j] = (char)(i + '0'); v.bin[j] = (char)(i + '0');
} }

36
examples/c/schemaless.c
View File

@ -1,32 +1,17 @@
#include "../../../include/client/taos.h" #include "taos.h"
#include "os.h"
#include "taoserror.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <sys/time.h> #include <sys/time.h>
#include <time.h> #include <time.h>
#include <unistd.h> #include <unistd.h>
#include <inttypes.h>
#include <string.h>
int numSuperTables = 8; int numSuperTables = 8;
int numChildTables = 4; int numChildTables = 4;
int numRowsPerChildTable = 2048; int numRowsPerChildTable = 2048;
void shuffle(char**lines, size_t n)
{
if (n > 1)
{
size_t i;
for (i = 0; i < n - 1; i++)
{
size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1);
char* t = lines[j];
lines[j] = lines[i];
lines[i] = t;
}
}
}
static int64_t getTimeInUs() { static int64_t getTimeInUs() {
struct timeval systemTime; struct timeval systemTime;
gettimeofday(&systemTime, NULL); gettimeofday(&systemTime, NULL);
@ -46,7 +31,7 @@ int main(int argc, char* argv[]) {
exit(1); exit(1);
} }
char* info = taos_get_server_info(taos); const char* info = taos_get_server_info(taos);
printf("server info: %s\n", info); printf("server info: %s\n", info);
info = taos_get_client_info(taos); info = taos_get_client_info(taos);
printf("client info: %s\n", info); printf("client info: %s\n", info);
@ -61,9 +46,10 @@ int main(int argc, char* argv[]) {
time_t ct = time(0); time_t ct = time(0);
int64_t ts = ct * 1000; int64_t ts = ct * 1000;
char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %" PRId64;
char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); int lineNum = numSuperTables * numChildTables * numRowsPerChildTable;
char** lines = calloc((size_t)lineNum, sizeof(char*));
int l = 0; int l = 0;
for (int i = 0; i < numSuperTables; ++i) { for (int i = 0; i < numSuperTables; ++i) {
for (int j = 0; j < numChildTables; ++j) { for (int j = 0; j < numChildTables; ++j) {
@ -75,13 +61,13 @@ int main(int argc, char* argv[]) {
} }
} }
} }
//shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable);
printf("%s\n", "begin taos_insert_lines"); printf("%s\n", "begin taos_insert_lines");
int64_t begin = getTimeInUs(); int64_t begin = getTimeInUs();
int32_t code = taos_insert_lines(taos, lines, numSuperTables * numChildTables * numRowsPerChildTable); TAOS_RES *res = taos_schemaless_insert(taos, lines, lineNum, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS);
int64_t end = getTimeInUs(); int64_t end = getTimeInUs();
printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin); printf("code: %s. time used: %" PRId64 "\n", taos_errstr(res), end-begin);
taos_free_result(res);
return 0; return 0;
} }

7
examples/c/stream_demo.c
View File

@ -108,10 +108,13 @@ int32_t create_stream() {
} }
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
int code;
if (argc > 1) { if (argc > 1) {
printf("env init\n"); printf("env init\n");
code = init_env(); int code = init_env();
if (code) {
return code;
}
} }
create_stream(); create_stream();
} }

20
examples/c/tmq.c
View File

@ -21,9 +21,6 @@
#include "taos.h" #include "taos.h"
static int running = 1; static int running = 1;
static char dbName[64] = "tmqdb";
static char stbName[64] = "stb";
static char topicName[64] = "topicname";
static int32_t msg_process(TAOS_RES* msg) { static int32_t msg_process(TAOS_RES* msg) {
char buf[1024]; char buf[1024];
@ -43,7 +40,7 @@ static int32_t msg_process(TAOS_RES* msg) {
TAOS_FIELD* fields = taos_fetch_fields(msg); TAOS_FIELD* fields = taos_fetch_fields(msg);
int32_t numOfFields = taos_field_count(msg); int32_t numOfFields = taos_field_count(msg);
int32_t* length = taos_fetch_lengths(msg); //int32_t* length = taos_fetch_lengths(msg);
int32_t precision = taos_result_precision(msg); int32_t precision = taos_result_precision(msg);
rows++; rows++;
taos_print_row(buf, row, fields, numOfFields); taos_print_row(buf, row, fields, numOfFields);
@ -62,6 +59,13 @@ static int32_t init_env() {
TAOS_RES* pRes; TAOS_RES* pRes;
// drop database if exists // drop database if exists
printf("create database\n"); printf("create database\n");
pRes = taos_query(pConn, "drop topic topicname");
if (taos_errno(pRes) != 0) {
printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes));
return -1;
}
taos_free_result(pRes);
pRes = taos_query(pConn, "drop database if exists tmqdb"); pRes = taos_query(pConn, "drop database if exists tmqdb");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes));
@ -249,7 +253,7 @@ int main(int argc, char* argv[]) {
tmq_t* tmq = build_consumer(); tmq_t* tmq = build_consumer();
if (NULL == tmq) { if (NULL == tmq) {
fprintf(stderr, "%% build_consumer() fail!\n"); fprintf(stderr, "build_consumer() fail!\n");
return -1; return -1;
} }
@ -259,7 +263,7 @@ int main(int argc, char* argv[]) {
} }
if ((code = tmq_subscribe(tmq, topic_list))) { if ((code = tmq_subscribe(tmq, topic_list))) {
fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); fprintf(stderr, "Failed to tmq_subscribe(): %s\n", tmq_err2str(code));
} }
tmq_list_destroy(topic_list); tmq_list_destroy(topic_list);
@ -267,9 +271,9 @@ int main(int argc, char* argv[]) {
code = tmq_consumer_close(tmq); code = tmq_consumer_close(tmq);
if (code) { if (code) {
fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); fprintf(stderr, "Failed to close consumer: %s\n", tmq_err2str(code));
} else { } else {
fprintf(stderr, "%% Consumer closed\n"); fprintf(stderr, "Consumer closed\n");
} }
return 0; return 0;

1
include/common/tcommon.h
View File

@ -195,6 +195,7 @@ typedef struct SDataBlockInfo {
uint32_t capacity; uint32_t capacity;
SBlockID id; SBlockID id;
int16_t hasVarCol; int16_t hasVarCol;
int16_t dataLoad; // denote if the data is loaded or not
// TODO: optimize and remove following // TODO: optimize and remove following
int64_t version; // used for stream, and need serialization int64_t version; // used for stream, and need serialization

2
include/common/tmsg.h
View File

@ -68,7 +68,7 @@ typedef uint16_t tmsg_t;
static inline bool vnodeIsMsgBlock(tmsg_t type) { static inline bool vnodeIsMsgBlock(tmsg_t type) {
return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) ||
(type == TDMT_VND_UPDATE_TAG_VAL); (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM);
} }
static inline bool syncUtilUserCommit(tmsg_t msgType) { static inline bool syncUtilUserCommit(tmsg_t msgType) {

478
include/common/ttokendef.h
View File

@ -94,245 +94,245 @@
#define TK_TSDB_PAGESIZE 76 #define TK_TSDB_PAGESIZE 76
#define TK_PRECISION 77 #define TK_PRECISION 77
#define TK_REPLICA 78 #define TK_REPLICA 78
#define TK_STRICT 79 #define TK_VGROUPS 79
#define TK_VGROUPS 80 #define TK_SINGLE_STABLE 80
#define TK_SINGLE_STABLE 81 #define TK_RETENTIONS 81
#define TK_RETENTIONS 82 #define TK_SCHEMALESS 82
#define TK_SCHEMALESS 83 #define TK_WAL_LEVEL 83
#define TK_WAL_LEVEL 84 #define TK_WAL_FSYNC_PERIOD 84
#define TK_WAL_FSYNC_PERIOD 85 #define TK_WAL_RETENTION_PERIOD 85
#define TK_WAL_RETENTION_PERIOD 86 #define TK_WAL_RETENTION_SIZE 86
#define TK_WAL_RETENTION_SIZE 87 #define TK_WAL_ROLL_PERIOD 87
#define TK_WAL_ROLL_PERIOD 88 #define TK_WAL_SEGMENT_SIZE 88
#define TK_WAL_SEGMENT_SIZE 89 #define TK_STT_TRIGGER 89
#define TK_STT_TRIGGER 90 #define TK_TABLE_PREFIX 90
#define TK_TABLE_PREFIX 91 #define TK_TABLE_SUFFIX 91
#define TK_TABLE_SUFFIX 92 #define TK_NK_COLON 92
#define TK_NK_COLON 93 #define TK_MAX_SPEED 93
#define TK_MAX_SPEED 94 #define TK_TABLE 94
#define TK_TABLE 95 #define TK_NK_LP 95
#define TK_NK_LP 96 #define TK_NK_RP 96
#define TK_NK_RP 97 #define TK_STABLE 97
#define TK_STABLE 98 #define TK_ADD 98
#define TK_ADD 99 #define TK_COLUMN 99
#define TK_COLUMN 100 #define TK_MODIFY 100
#define TK_MODIFY 101 #define TK_RENAME 101
#define TK_RENAME 102 #define TK_TAG 102
#define TK_TAG 103 #define TK_SET 103
#define TK_SET 104 #define TK_NK_EQ 104
#define TK_NK_EQ 105 #define TK_USING 105
#define TK_USING 106 #define TK_TAGS 106
#define TK_TAGS 107 #define TK_COMMENT 107
#define TK_COMMENT 108 #define TK_BOOL 108
#define TK_BOOL 109 #define TK_TINYINT 109
#define TK_TINYINT 110 #define TK_SMALLINT 110
#define TK_SMALLINT 111 #define TK_INT 111
#define TK_INT 112 #define TK_INTEGER 112
#define TK_INTEGER 113 #define TK_BIGINT 113
#define TK_BIGINT 114 #define TK_FLOAT 114
#define TK_FLOAT 115 #define TK_DOUBLE 115
#define TK_DOUBLE 116 #define TK_BINARY 116
#define TK_BINARY 117 #define TK_TIMESTAMP 117
#define TK_TIMESTAMP 118 #define TK_NCHAR 118
#define TK_NCHAR 119 #define TK_UNSIGNED 119
#define TK_UNSIGNED 120 #define TK_JSON 120
#define TK_JSON 121 #define TK_VARCHAR 121
#define TK_VARCHAR 122 #define TK_MEDIUMBLOB 122
#define TK_MEDIUMBLOB 123 #define TK_BLOB 123
#define TK_BLOB 124 #define TK_VARBINARY 124
#define TK_VARBINARY 125 #define TK_DECIMAL 125
#define TK_DECIMAL 126 #define TK_MAX_DELAY 126
#define TK_MAX_DELAY 127 #define TK_WATERMARK 127
#define TK_WATERMARK 128 #define TK_ROLLUP 128
#define TK_ROLLUP 129 #define TK_TTL 129
#define TK_TTL 130 #define TK_SMA 130
#define TK_SMA 131 #define TK_DELETE_MARK 131
#define TK_DELETE_MARK 132 #define TK_FIRST 132
#define TK_FIRST 133 #define TK_LAST 133
#define TK_LAST 134 #define TK_SHOW 134
#define TK_SHOW 135 #define TK_PRIVILEGES 135
#define TK_PRIVILEGES 136 #define TK_DATABASES 136
#define TK_DATABASES 137 #define TK_TABLES 137
#define TK_TABLES 138 #define TK_STABLES 138
#define TK_STABLES 139 #define TK_MNODES 139
#define TK_MNODES 140 #define TK_QNODES 140
#define TK_QNODES 141 #define TK_FUNCTIONS 141
#define TK_FUNCTIONS 142 #define TK_INDEXES 142
#define TK_INDEXES 143 #define TK_ACCOUNTS 143
#define TK_ACCOUNTS 144 #define TK_APPS 144
#define TK_APPS 145 #define TK_CONNECTIONS 145
#define TK_CONNECTIONS 146 #define TK_LICENCES 146
#define TK_LICENCES 147 #define TK_GRANTS 147
#define TK_GRANTS 148 #define TK_QUERIES 148
#define TK_QUERIES 149 #define TK_SCORES 149
#define TK_SCORES 150 #define TK_TOPICS 150
#define TK_TOPICS 151 #define TK_VARIABLES 151
#define TK_VARIABLES 152 #define TK_CLUSTER 152
#define TK_CLUSTER 153 #define TK_BNODES 153
#define TK_BNODES 154 #define TK_SNODES 154
#define TK_SNODES 155 #define TK_TRANSACTIONS 155
#define TK_TRANSACTIONS 156 #define TK_DISTRIBUTED 156
#define TK_DISTRIBUTED 157 #define TK_CONSUMERS 157
#define TK_CONSUMERS 158 #define TK_SUBSCRIPTIONS 158
#define TK_SUBSCRIPTIONS 159 #define TK_VNODES 159
#define TK_VNODES 160 #define TK_LIKE 160
#define TK_LIKE 161 #define TK_TBNAME 161
#define TK_TBNAME 162 #define TK_QTAGS 162
#define TK_QTAGS 163 #define TK_AS 163
#define TK_AS 164 #define TK_INDEX 164
#define TK_INDEX 165 #define TK_FUNCTION 165
#define TK_FUNCTION 166 #define TK_INTERVAL 166
#define TK_INTERVAL 167 #define TK_TOPIC 167
#define TK_TOPIC 168 #define TK_WITH 168
#define TK_WITH 169 #define TK_META 169
#define TK_META 170 #define TK_CONSUMER 170
#define TK_CONSUMER 171 #define TK_GROUP 171
#define TK_GROUP 172 #define TK_DESC 172
#define TK_DESC 173 #define TK_DESCRIBE 173
#define TK_DESCRIBE 174 #define TK_RESET 174
#define TK_RESET 175 #define TK_QUERY 175
#define TK_QUERY 176 #define TK_CACHE 176
#define TK_CACHE 177 #define TK_EXPLAIN 177
#define TK_EXPLAIN 178 #define TK_ANALYZE 178
#define TK_ANALYZE 179 #define TK_VERBOSE 179
#define TK_VERBOSE 180 #define TK_NK_BOOL 180
#define TK_NK_BOOL 181 #define TK_RATIO 181
#define TK_RATIO 182 #define TK_NK_FLOAT 182
#define TK_NK_FLOAT 183 #define TK_OUTPUTTYPE 183
#define TK_OUTPUTTYPE 184 #define TK_AGGREGATE 184
#define TK_AGGREGATE 185 #define TK_BUFSIZE 185
#define TK_BUFSIZE 186 #define TK_STREAM 186
#define TK_STREAM 187 #define TK_INTO 187
#define TK_INTO 188 #define TK_TRIGGER 188
#define TK_TRIGGER 189 #define TK_AT_ONCE 189
#define TK_AT_ONCE 190 #define TK_WINDOW_CLOSE 190
#define TK_WINDOW_CLOSE 191 #define TK_IGNORE 191
#define TK_IGNORE 192 #define TK_EXPIRED 192
#define TK_EXPIRED 193 #define TK_FILL_HISTORY 193
#define TK_FILL_HISTORY 194 #define TK_SUBTABLE 194
#define TK_SUBTABLE 195 #define TK_KILL 195
#define TK_KILL 196 #define TK_CONNECTION 196
#define TK_CONNECTION 197 #define TK_TRANSACTION 197
#define TK_TRANSACTION 198 #define TK_BALANCE 198
#define TK_BALANCE 199 #define TK_VGROUP 199
#define TK_VGROUP 200 #define TK_MERGE 200
#define TK_MERGE 201 #define TK_REDISTRIBUTE 201
#define TK_REDISTRIBUTE 202 #define TK_SPLIT 202
#define TK_SPLIT 203 #define TK_DELETE 203
#define TK_DELETE 204 #define TK_INSERT 204
#define TK_INSERT 205 #define TK_NULL 205
#define TK_NULL 206 #define TK_NK_QUESTION 206
#define TK_NK_QUESTION 207 #define TK_NK_ARROW 207
#define TK_NK_ARROW 208 #define TK_ROWTS 208
#define TK_ROWTS 209 #define TK_QSTART 209
#define TK_QSTART 210 #define TK_QEND 210
#define TK_QEND 211 #define TK_QDURATION 211
#define TK_QDURATION 212 #define TK_WSTART 212
#define TK_WSTART 213 #define TK_WEND 213
#define TK_WEND 214 #define TK_WDURATION 214
#define TK_WDURATION 215 #define TK_IROWTS 215
#define TK_IROWTS 216 #define TK_ISFILLED 216
#define TK_ISFILLED 217 #define TK_CAST 217
#define TK_CAST 218 #define TK_NOW 218
#define TK_NOW 219 #define TK_TODAY 219
#define TK_TODAY 220 #define TK_TIMEZONE 220
#define TK_TIMEZONE 221 #define TK_CLIENT_VERSION 221
#define TK_CLIENT_VERSION 222 #define TK_SERVER_VERSION 222
#define TK_SERVER_VERSION 223 #define TK_SERVER_STATUS 223
#define TK_SERVER_STATUS 224 #define TK_CURRENT_USER 224
#define TK_CURRENT_USER 225 #define TK_COUNT 225
#define TK_COUNT 226 #define TK_LAST_ROW 226
#define TK_LAST_ROW 227 #define TK_CASE 227
#define TK_CASE 228 #define TK_END 228
#define TK_END 229 #define TK_WHEN 229
#define TK_WHEN 230 #define TK_THEN 230
#define TK_THEN 231 #define TK_ELSE 231
#define TK_ELSE 232 #define TK_BETWEEN 232
#define TK_BETWEEN 233 #define TK_IS 233
#define TK_IS 234 #define TK_NK_LT 234
#define TK_NK_LT 235 #define TK_NK_GT 235
#define TK_NK_GT 236 #define TK_NK_LE 236
#define TK_NK_LE 237 #define TK_NK_GE 237
#define TK_NK_GE 238 #define TK_NK_NE 238
#define TK_NK_NE 239 #define TK_MATCH 239
#define TK_MATCH 240 #define TK_NMATCH 240
#define TK_NMATCH 241 #define TK_CONTAINS 241
#define TK_CONTAINS 242 #define TK_IN 242
#define TK_IN 243 #define TK_JOIN 243
#define TK_JOIN 244 #define TK_INNER 244
#define TK_INNER 245 #define TK_SELECT 245
#define TK_SELECT 246 #define TK_DISTINCT 246
#define TK_DISTINCT 247 #define TK_WHERE 247
#define TK_WHERE 248 #define TK_PARTITION 248
#define TK_PARTITION 249 #define TK_BY 249
#define TK_BY 250 #define TK_SESSION 250
#define TK_SESSION 251 #define TK_STATE_WINDOW 251
#define TK_STATE_WINDOW 252 #define TK_EVENT_WINDOW 252
#define TK_EVENT_WINDOW 253 #define TK_START 253
#define TK_START 254 #define TK_SLIDING 254
#define TK_SLIDING 255 #define TK_FILL 255
#define TK_FILL 256 #define TK_VALUE 256
#define TK_VALUE 257 #define TK_NONE 257
#define TK_NONE 258 #define TK_PREV 258
#define TK_PREV 259 #define TK_LINEAR 259
#define TK_LINEAR 260 #define TK_NEXT 260
#define TK_NEXT 261 #define TK_HAVING 261
#define TK_HAVING 262 #define TK_RANGE 262
#define TK_RANGE 263 #define TK_EVERY 263
#define TK_EVERY 264 #define TK_ORDER 264
#define TK_ORDER 265 #define TK_SLIMIT 265
#define TK_SLIMIT 266 #define TK_SOFFSET 266
#define TK_SOFFSET 267 #define TK_LIMIT 267
#define TK_LIMIT 268 #define TK_OFFSET 268
#define TK_OFFSET 269 #define TK_ASC 269
#define TK_ASC 270 #define TK_NULLS 270
#define TK_NULLS 271 #define TK_ABORT 271
#define TK_ABORT 272 #define TK_AFTER 272
#define TK_AFTER 273 #define TK_ATTACH 273
#define TK_ATTACH 274 #define TK_BEFORE 274
#define TK_BEFORE 275 #define TK_BEGIN 275
#define TK_BEGIN 276 #define TK_BITAND 276
#define TK_BITAND 277 #define TK_BITNOT 277
#define TK_BITNOT 278 #define TK_BITOR 278
#define TK_BITOR 279 #define TK_BLOCKS 279
#define TK_BLOCKS 280 #define TK_CHANGE 280
#define TK_CHANGE 281 #define TK_COMMA 281
#define TK_COMMA 282 #define TK_COMPACT 282
#define TK_COMPACT 283 #define TK_CONCAT 283
#define TK_CONCAT 284 #define TK_CONFLICT 284
#define TK_CONFLICT 285 #define TK_COPY 285
#define TK_COPY 286 #define TK_DEFERRED 286
#define TK_DEFERRED 287 #define TK_DELIMITERS 287
#define TK_DELIMITERS 288 #define TK_DETACH 288
#define TK_DETACH 289 #define TK_DIVIDE 289
#define TK_DIVIDE 290 #define TK_DOT 290
#define TK_DOT 291 #define TK_EACH 291
#define TK_EACH 292 #define TK_FAIL 292
#define TK_FAIL 293 #define TK_FILE 293
#define TK_FILE 294 #define TK_FOR 294
#define TK_FOR 295 #define TK_GLOB 295
#define TK_GLOB 296 #define TK_ID 296
#define TK_ID 297 #define TK_IMMEDIATE 297
#define TK_IMMEDIATE 298 #define TK_IMPORT 298
#define TK_IMPORT 299 #define TK_INITIALLY 299
#define TK_INITIALLY 300 #define TK_INSTEAD 300
#define TK_INSTEAD 301 #define TK_ISNULL 301
#define TK_ISNULL 302 #define TK_KEY 302
#define TK_KEY 303 #define TK_MODULES 303
#define TK_MODULES 304 #define TK_NK_BITNOT 304
#define TK_NK_BITNOT 305 #define TK_NK_SEMI 305
#define TK_NK_SEMI 306 #define TK_NOTNULL 306
#define TK_NOTNULL 307 #define TK_OF 307
#define TK_OF 308 #define TK_PLUS 308
#define TK_PLUS 309 #define TK_PRIVILEGE 309
#define TK_PRIVILEGE 310 #define TK_RAISE 310
#define TK_RAISE 311 #define TK_REPLACE 311
#define TK_REPLACE 312 #define TK_RESTRICT 312
#define TK_RESTRICT 313 #define TK_ROW 313
#define TK_ROW 314 #define TK_SEMI 314
#define TK_SEMI 315 #define TK_STAR 315
#define TK_STAR 316 #define TK_STATEMENT 316
#define TK_STATEMENT 317 #define TK_STRICT 317
#define TK_STRING 318 #define TK_STRING 318
#define TK_TIMES 319 #define TK_TIMES 319
#define TK_UPDATE 320 #define TK_UPDATE 320

22
include/libs/qcom/query.h
View File

@ -58,8 +58,7 @@ typedef enum {
#define QUERY_RSP_POLICY_QUICK 1 #define QUERY_RSP_POLICY_QUICK 1
#define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0)
#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0)
typedef struct STableComInfo { typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema uint8_t numOfTags; // the number of tags in schema
@ -128,7 +127,7 @@ typedef struct SDBVgInfo {
int8_t hashMethod; int8_t hashMethod;
int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT
int64_t stateTs; int64_t stateTs;
SHashObj* vgHash; // key:vgId, value:SVgroupInfo SHashObj* vgHash; // key:vgId, value:SVgroupInfo
SArray* vgArray; SArray* vgArray;
} SDBVgInfo; } SDBVgInfo;
@ -262,23 +261,26 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
(NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \
NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code))
#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED || (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) #define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) \
#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || \
#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND) (_code) == TSDB_CODE_VND_STOPPED || (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING)
#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) \
((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR)
#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND)
#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL)
#define NEED_REDIRECT_ERROR(_code) \ #define NEED_REDIRECT_ERROR(_code) \
(NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code)) SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB) (_type) == TDMT_MND_DROP_STB)
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
(SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code)) (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \
SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
#define REQUEST_TOTAL_EXEC_TIMES 2 #define REQUEST_TOTAL_EXEC_TIMES 2

3
include/libs/sync/sync.h
View File

@ -230,7 +230,7 @@ int64_t syncOpen(SSyncInfo* pSyncInfo);
int32_t syncStart(int64_t rid); int32_t syncStart(int64_t rid);
void syncStop(int64_t rid); void syncStop(int64_t rid);
void syncPreStop(int64_t rid); void syncPreStop(int64_t rid);
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq);
int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg);
int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg);
int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex);
@ -240,6 +240,7 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm);
bool syncIsReadyForRead(int64_t rid); bool syncIsReadyForRead(int64_t rid);
bool syncSnapshotSending(int64_t rid); bool syncSnapshotSending(int64_t rid);
bool syncSnapshotRecving(int64_t rid); bool syncSnapshotRecving(int64_t rid);
int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq);
SSyncState syncGetState(int64_t rid); SSyncState syncGetState(int64_t rid);
void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet);

14
include/libs/transport/trpc.h
View File

@ -77,14 +77,12 @@ typedef void (*RpcDfp)(void *ahandle);
typedef struct SRpcInit { typedef struct SRpcInit {
char localFqdn[TSDB_FQDN_LEN]; char localFqdn[TSDB_FQDN_LEN];
uint16_t localPort; // local port uint16_t localPort; // local port
char *label; // for debug purpose char *label; // for debug purpose
int32_t numOfThreads; // number of threads to handle connections int32_t numOfThreads; // number of threads to handle connections
int32_t sessions; // number of sessions allowed int32_t sessions; // number of sessions allowed
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
int32_t idleTime; // milliseconds, 0 means idle timer is disabled int32_t idleTime; // milliseconds, 0 means idle timer is disabled
int32_t retryLimit; // retry limit
int32_t retryInterval; // retry interval ms
int32_t retryMinInterval; // retry init interval int32_t retryMinInterval; // retry init interval
int32_t retryStepFactor; // retry interval factor int32_t retryStepFactor; // retry interval factor

2
include/os/osEnv.h
View File

@ -36,7 +36,7 @@ extern int64_t tsStreamMax;
extern float tsNumOfCores; extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB; extern int64_t tsTotalMemoryKB;
extern char *tsProcPath; extern char *tsProcPath;
extern char tsSIMDEnable; extern char tsSIMDBuiltins;
extern char tsSSE42Enable; extern char tsSSE42Enable;
extern char tsAVXEnable; extern char tsAVXEnable;
extern char tsAVX2Enable; extern char tsAVX2Enable;

1
include/util/taoserror.h
View File

@ -342,6 +342,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2)
#define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) #define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3)
#define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4) #define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4)
#define TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB TAOS_DEF_ERROR_CODE(0, 0x03F5)
// mnode-sma // mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)

5
include/util/tdef.h
View File

@ -337,7 +337,7 @@ typedef enum ELogicConditionType {
#define TSDB_DB_STRICT_ON_STR "on" #define TSDB_DB_STRICT_ON_STR "on"
#define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_OFF 0
#define TSDB_DB_STRICT_ON 1 #define TSDB_DB_STRICT_ON 1
#define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_OFF #define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_ON
#define TSDB_CACHE_MODEL_STR_LEN sizeof(TSDB_CACHE_MODEL_LAST_VALUE_STR) #define TSDB_CACHE_MODEL_STR_LEN sizeof(TSDB_CACHE_MODEL_LAST_VALUE_STR)
#define TSDB_CACHE_MODEL_NONE_STR "none" #define TSDB_CACHE_MODEL_NONE_STR "none"
#define TSDB_CACHE_MODEL_LAST_ROW_STR "last_row" #define TSDB_CACHE_MODEL_LAST_ROW_STR "last_row"
@ -497,6 +497,9 @@ enum {
// sort page size by default // sort page size by default
#define DEFAULT_PAGESIZE 4096 #define DEFAULT_PAGESIZE 4096
#define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 10
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

5
source/client/src/clientEnv.c
View File

@ -146,8 +146,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
rpcInit.dfp = destroyAhandle; rpcInit.dfp = destroyAhandle;
rpcInit.retryLimit = tsRpcRetryLimit;
rpcInit.retryInterval = tsRpcRetryInterval;
rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryMinInterval = tsRedirectPeriod;
rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryStepFactor = tsRedirectFactor;
rpcInit.retryMaxInterval = tsRedirectMaxPeriod; rpcInit.retryMaxInterval = tsRedirectMaxPeriod;
@ -232,7 +231,7 @@ void destroyTscObj(void *pObj) {
pTscObj->pAppInfo->numOfConns); pTscObj->pAppInfo->numOfConns);
// In any cases, we should not free app inst here. Or an race condition rises. // In any cases, we should not free app inst here. Or an race condition rises.
/*int64_t connNum = */atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); /*int64_t connNum = */ atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
taosThreadMutexDestroy(&pTscObj->mutex); taosThreadMutexDestroy(&pTscObj->mutex);
taosMemoryFree(pTscObj); taosMemoryFree(pTscObj);

17
source/client/src/clientImpl.c
View File

@ -190,8 +190,8 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
(*pRequest)->body.param = param; (*pRequest)->body.param = param;
STscObj* pTscObj = (*pRequest)->pTscObj; STscObj* pTscObj = (*pRequest)->pTscObj;
int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
sizeof((*pRequest)->self)); sizeof((*pRequest)->self));
if (err) { if (err) {
tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
(*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
@ -1646,7 +1646,8 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) {
static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) {
char* p = (char*)pResultInfo->pData; char* p = (char*)pResultInfo->pData;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
// length |
int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t len = getVersion1BlockMetaSize(p, numOfCols);
int32_t* colLength = (int32_t*)(p + len); int32_t* colLength = (int32_t*)(p + len);
len += sizeof(int32_t) * numOfCols; len += sizeof(int32_t) * numOfCols;
@ -1972,8 +1973,6 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
rpcInit.user = "_dnd"; rpcInit.user = "_dnd";
rpcInit.retryLimit = tsRpcRetryLimit;
rpcInit.retryInterval = tsRpcRetryInterval;
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
if (clientRpc == NULL) { if (clientRpc == NULL) {
@ -2295,14 +2294,14 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
tsem_wait(&param->sem); tsem_wait(&param->sem);
SRequestObj *pRequest = NULL; SRequestObj* pRequest = NULL;
if (param->pRequest != NULL) { if (param->pRequest != NULL) {
param->pRequest->syncQuery = true; param->pRequest->syncQuery = true;
pRequest = param->pRequest; pRequest = param->pRequest;
} else { } else {
taosMemoryFree(param); taosMemoryFree(param);
} }
return pRequest; return pRequest;
} }
@ -2318,13 +2317,13 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid); taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
tsem_wait(&param->sem); tsem_wait(&param->sem);
SRequestObj *pRequest = NULL; SRequestObj* pRequest = NULL;
if (param->pRequest != NULL) { if (param->pRequest != NULL) {
param->pRequest->syncQuery = true; param->pRequest->syncQuery = true;
pRequest = param->pRequest; pRequest = param->pRequest;
} else { } else {
taosMemoryFree(param); taosMemoryFree(param);
} }
return pRequest; return pRequest;
} }

6
source/client/src/clientTmq.c
View File

@ -1035,7 +1035,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SCMSubscribeReq req = {0}; SCMSubscribeReq req = {0};
int32_t code = -1; int32_t code = -1;
tscDebug("call tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); tscDebug("tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz);
req.consumerId = tmq->consumerId; req.consumerId = tmq->consumerId;
tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.clientId, tmq->clientId, 256);
@ -1043,7 +1043,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
req.topicNames = taosArrayInit(sz, sizeof(void*)); req.topicNames = taosArrayInit(sz, sizeof(void*));
if (req.topicNames == NULL) goto FAIL; if (req.topicNames == NULL) goto FAIL;
tscDebug("call tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); tscDebug("tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char* topic = taosArrayGetP(container, i); char* topic = taosArrayGetP(container, i);
@ -1570,7 +1570,6 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) {
} }
int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
/*tscDebug("call poll");*/
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
@ -1794,7 +1793,6 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
} }
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
/*tscDebug("call poll1");*/
void* rspObj; void* rspObj;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();

12
source/common/src/tdatablock.c
View File

@ -358,7 +358,11 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSiz
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) {
if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { if (pDataBlock->info.rows > 0) {
// ASSERT(pDataBlock->info.dataLoad == 1);
}
if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) {
return 0; return 0;
} }
@ -1157,13 +1161,14 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
} }
pInfo->rows = 0; pInfo->rows = 0;
pInfo->dataLoad = 0;
pInfo->window.ekey = 0; pInfo->window.ekey = 0;
pInfo->window.skey = 0; pInfo->window.skey = 0;
} }
// todo temporarily disable it // todo temporarily disable it
static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) {
ASSERT(numOfRows > 0 /*&& pBlockInfo->capacity >= pBlockInfo->rows*/); ASSERT(numOfRows > 0);
if (numOfRows <= pBlockInfo->capacity) { if (numOfRows <= pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1220,7 +1225,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo*
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) {
pColumn->hasNull = false; pColumn->hasNull = false;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
@ -2427,6 +2432,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
pStart += colLen[i]; pStart += colLen[i];
} }
pBlock->info.dataLoad = 1;
pBlock->info.rows = numOfRows; pBlock->info.rows = numOfRows;
ASSERT(pStart - pData == dataLen); ASSERT(pStart - pData == dataLen);
return pStart; return pStart;

22
source/common/src/tglobal.c
View File

@ -180,8 +180,6 @@ int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
char tsUdfdLdLibPath[512] = ""; char tsUdfdLdLibPath[512] = "";
int32_t tsRpcRetryLimit = 100;
int32_t tsRpcRetryInterval = 15;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -203,7 +201,9 @@ int32_t taosSetTfsCfg(SConfig *pCfg) {
int32_t taosSetTfsCfg(SConfig *pCfg); int32_t taosSetTfsCfg(SConfig *pCfg);
#endif #endif
struct SConfig *taosGetCfg() { return tsCfg; } struct SConfig *taosGetCfg() {
return tsCfg;
}
static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile,
char *apolloUrl) { char *apolloUrl) {
@ -313,8 +313,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = tsNumOfCores / 2;
@ -341,7 +339,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1; if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1;
if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1; if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1;
if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1; if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1;
if (cfgAddBool(pCfg, "SIMD-Supported", tsSIMDEnable, 0) != 0) return -1; if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
@ -457,9 +455,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1;
if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1;
GRANT_CFG_ADD; GRANT_CFG_ADD;
return 0; return 0;
} }
@ -674,8 +669,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval;
tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32;
tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
return 0; return 0;
} }
@ -738,10 +731,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN);
tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32;
tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32;
tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32;
tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
@ -761,9 +750,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
if (tsQueryBufferSize >= 0) { if (tsQueryBufferSize >= 0) {
tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
} }
tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32;
tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32;
GRANT_CFG_GET; GRANT_CFG_GET;
return 0; return 0;
} }

2
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -38,6 +38,8 @@ typedef struct SVnodeMgmt {
TdThreadRwlock lock; TdThreadRwlock lock;
SVnodesStat state; SVnodesStat state;
STfs *pTfs; STfs *pTfs;
TdThread thread;
bool stop;
} SVnodeMgmt; } SVnodeMgmt;
typedef struct { typedef struct {

62
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -334,6 +334,62 @@ static void vmCleanup(SVnodeMgmt *pMgmt) {
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
} }
static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
int32_t numOfVnodes = 0;
SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes);
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = ppVnodes[i];
vnodeSyncCheckTimeout(pVnode->pImpl);
vmReleaseVnode(pMgmt, pVnode);
}
if (ppVnodes != NULL) {
taosMemoryFree(ppVnodes);
}
}
static void *vmThreadFp(void *param) {
SVnodeMgmt *pMgmt = param;
int64_t lastTime = 0;
setThreadName("vnode-timer");
while (1) {
lastTime++;
taosMsleep(100);
if (pMgmt->stop) break;
if (lastTime % 10 != 0) continue;
int64_t sec = lastTime / 10;
if (sec % (VNODE_TIMEOUT_SEC / 2) == 0) {
vmCheckSyncTimeout(pMgmt);
}
}
return NULL;
}
static int32_t vmInitTimer(SVnodeMgmt *pMgmt) {
TdThreadAttr thAttr;
taosThreadAttrInit(&thAttr);
taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) {
dError("failed to create vnode timer thread since %s", strerror(errno));
return -1;
}
taosThreadAttrDestroy(&thAttr);
return 0;
}
static void vmCleanupTimer(SVnodeMgmt *pMgmt) {
pMgmt->stop = true;
if (taosCheckPthreadValid(pMgmt->thread)) {
taosThreadJoin(pMgmt->thread, NULL);
taosThreadClear(&pMgmt->thread);
}
}
static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
int32_t code = -1; int32_t code = -1;
@ -510,12 +566,10 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) {
taosMemoryFree(ppVnodes); taosMemoryFree(ppVnodes);
} }
return 0; return vmInitTimer(pMgmt);
} }
static void vmStop(SVnodeMgmt *pMgmt) { static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); }
// process inside the vnode
}
SMgmtFunc vmGetMgmtFunc() { SMgmtFunc vmGetMgmtFunc() {
SMgmtFunc mgmtFunc = {0}; SMgmtFunc mgmtFunc = {0};

3
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -214,6 +214,9 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) {
case SNODE: case SNODE:
terrno = TSDB_CODE_SNODE_NOT_FOUND; terrno = TSDB_CODE_SNODE_NOT_FOUND;
break; break;
case VNODE:
terrno = TSDB_CODE_VND_STOPPED;
break;
default: default:
terrno = TSDB_CODE_APP_IS_STOPPING; terrno = TSDB_CODE_APP_IS_STOPPING;
break; break;

5
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -145,7 +145,8 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
if (pMsg == NULL) goto _OVER; if (pMsg == NULL) goto _OVER;
memcpy(pMsg, pRpc, sizeof(SRpcMsg)); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle, pRpc->contLen); dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle,
pRpc->contLen);
code = dmProcessNodeMsg(pWrapper, pMsg); code = dmProcessNodeMsg(pWrapper, pMsg);
@ -258,8 +259,6 @@ int32_t dmInitClient(SDnode *pDnode) {
rpcInit.rfp = rpcRfp; rpcInit.rfp = rpcRfp;
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
rpcInit.retryLimit = tsRpcRetryLimit;
rpcInit.retryInterval = tsRpcRetryInterval;
rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryMinInterval = tsRedirectPeriod;
rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryStepFactor = tsRedirectFactor;
rpcInit.retryMaxInterval = tsRedirectMaxPeriod; rpcInit.retryMaxInterval = tsRedirectMaxPeriod;

18
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -84,14 +84,16 @@ typedef struct {
} STelemMgmt; } STelemMgmt;
typedef struct { typedef struct {
tsem_t syncSem; tsem_t syncSem;
int64_t sync; int64_t sync;
int32_t errCode; int32_t errCode;
int32_t transId; int32_t transId;
SRWLatch lock; int32_t transSec;
int8_t selfIndex; int64_t transSeq;
int8_t numOfReplicas; TdThreadMutex lock;
SReplica replicas[TSDB_MAX_REPLICA]; int8_t selfIndex;
int8_t numOfReplicas;
SReplica replicas[TSDB_MAX_REPLICA];
} SSyncMgmt; } SSyncMgmt;
typedef struct { typedef struct {

1
source/dnode/mnode/impl/inc/mndSync.h
View File

@ -26,6 +26,7 @@ int32_t mndInitSync(SMnode *pMnode);
void mndCleanupSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode);
bool mndIsLeader(SMnode *pMnode); bool mndIsLeader(SMnode *pMnode);
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId);
void mndSyncCheckTimeout(SMnode *pMnode);
void mndSyncStart(SMnode *pMnode); void mndSyncStart(SMnode *pMnode);
void mndSyncStop(SMnode *pMnode); void mndSyncStop(SMnode *pMnode);

1
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -75,6 +75,7 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, voi
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname);
void mndTransSetSerial(STrans *pTrans); void mndTransSetSerial(STrans *pTrans);
void mndTransSetOper(STrans *pTrans, EOperType oper); void mndTransSetOper(STrans *pTrans, EOperType oper);
int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
int32_t mndTransProcessRsp(SRpcMsg *pRsp); int32_t mndTransProcessRsp(SRpcMsg *pRsp);

18
source/dnode/mnode/impl/src/mndDb.c
View File

@ -776,6 +776,8 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p
int32_t code = -1; int32_t code = -1;
mndTransSetDbName(pTrans, pOld->name, NULL); mndTransSetDbName(pTrans, pOld->name, NULL);
if (mndTrancCheckConflict(pMnode, pTrans) != 0) return -1;
if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER;
@ -835,12 +837,14 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
if (terrno != 0) code = terrno;
mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); mError("db:%s, failed to alter since %s", alterReq.db, terrstr());
} }
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
taosArrayDestroy(dbObj.cfg.pRetensions); taosArrayDestroy(dbObj.cfg.pRetensions);
terrno = code;
return code; return code;
} }
@ -1183,7 +1187,8 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs
int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); int32_t numOfTable = mndGetDBTableNum(pDb, pMnode);
if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable ||
pReq->stateTs < pDb->stateTs) {
mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos);
} }
@ -1298,21 +1303,22 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
SUseDbRsp usedbRsp = {0}; SUseDbRsp usedbRsp = {0};
if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) ||
(0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) {
memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN);
int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode);
if (pDbVgVersion->vgVersion < vgVersion) { if (pDbVgVersion->vgVersion < vgVersion) {
usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
usedbRsp.vgVersion = vgVersion++; usedbRsp.vgVersion = vgVersion++;
} else { } else {
usedbRsp.vgVersion = pDbVgVersion->vgVersion; usedbRsp.vgVersion = pDbVgVersion->vgVersion;
} }
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
taosArrayPush(batchUseRsp.pArray, &usedbRsp); taosArrayPush(batchUseRsp.pArray, &usedbRsp);
continue; continue;
} }
@ -1328,7 +1334,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs,
int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); int32_t numOfTable = mndGetDBTableNum(pDb, pMnode);
if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable &&
pDbVgVersion->stateTs == pDb->stateTs) { pDbVgVersion->stateTs == pDb->stateTs) {
mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64
" numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d",

1
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -672,6 +672,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port);
pDnode = mndAcquireDnodeByEp(pMnode, ep); pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode != NULL) { if (pDnode != NULL) {
terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
goto _OVER; goto _OVER;
} }

17
source/dnode/mnode/impl/src/mndMain.c
View File

@ -101,6 +101,7 @@ static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) {
} }
static void mndPullupTrans(SMnode *pMnode) { static void mndPullupTrans(SMnode *pMnode) {
mTrace("pullup trans msg");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
@ -110,6 +111,7 @@ static void mndPullupTrans(SMnode *pMnode) {
} }
static void mndPullupTtl(SMnode *pMnode) { static void mndPullupTtl(SMnode *pMnode) {
mTrace("pullup ttl");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen};
@ -117,6 +119,7 @@ static void mndPullupTtl(SMnode *pMnode) {
} }
static void mndCalMqRebalance(SMnode *pMnode) { static void mndCalMqRebalance(SMnode *pMnode) {
mTrace("calc mq rebalance");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
@ -143,6 +146,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) {
} }
static void mndPullupTelem(SMnode *pMnode) { static void mndPullupTelem(SMnode *pMnode) {
mTrace("pullup telem msg");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
@ -152,6 +156,7 @@ static void mndPullupTelem(SMnode *pMnode) {
} }
static void mndPullupGrant(SMnode *pMnode) { static void mndPullupGrant(SMnode *pMnode) {
mTrace("pullup grant msg");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
@ -162,6 +167,7 @@ static void mndPullupGrant(SMnode *pMnode) {
} }
static void mndIncreaseUpTime(SMnode *pMnode) { static void mndIncreaseUpTime(SMnode *pMnode) {
mTrace("increate uptime");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
if (pReq != NULL) { if (pReq != NULL) {
@ -213,6 +219,9 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs)
} }
static void mndCheckDnodeOffline(SMnode *pMnode) { static void mndCheckDnodeOffline(SMnode *pMnode) {
mTrace("check dnode offline");
if (mndAcquireRpc(pMnode) != 0) return;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int64_t curMs = taosGetTimestampMs(); int64_t curMs = taosGetTimestampMs();
@ -230,6 +239,8 @@ static void mndCheckDnodeOffline(SMnode *pMnode) {
sdbRelease(pSdb, pDnode); sdbRelease(pSdb, pDnode);
} }
mndReleaseRpc(pMnode);
} }
static void *mndThreadFp(void *param) { static void *mndThreadFp(void *param) {
@ -277,6 +288,10 @@ static void *mndThreadFp(void *param) {
if (sec % (tsStatusInterval * 5) == 0) { if (sec % (tsStatusInterval * 5) == 0) {
mndCheckDnodeOffline(pMnode); mndCheckDnodeOffline(pMnode);
} }
if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) {
mndSyncCheckTimeout(pMnode);
}
} }
return NULL; return NULL;
@ -648,7 +663,9 @@ _OVER:
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
SEpSet epSet = {0}; SEpSet epSet = {0};
int32_t tmpCode = terrno;
mndGetMnodeEpSet(pMnode, &epSet); mndGetMnodeEpSet(pMnode, &epSet);
terrno = tmpCode;
mGDebug( mGDebug(
"msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d "

14
source/dnode/mnode/impl/src/mndStream.c
View File

@ -164,7 +164,8 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) {
STREAM_DECODE_OVER: STREAM_DECODE_OVER:
taosMemoryFreeClear(buf); taosMemoryFreeClear(buf);
if (terrno != TSDB_CODE_SUCCESS) { if (terrno != TSDB_CODE_SUCCESS) {
mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, terrstr()); mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw,
terrstr());
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
} }
@ -624,6 +625,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
pDb = mndAcquireDb(pMnode, streamObj.sourceDb);
if (pDb->cfg.replications != 1) {
mError("stream source db must have only 1 replica, but %s has %d", pDb->name, pDb->cfg.replications);
terrno = TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB;
mndReleaseDb(pMnode, pDb);
pDb = NULL;
goto _OVER;
}
mndReleaseDb(pMnode, pDb);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
@ -680,7 +691,6 @@ _OVER:
} }
mndReleaseStream(pMnode, pStream); mndReleaseStream(pMnode, pStream);
mndReleaseDb(pMnode, pDb);
tFreeSCMCreateStreamReq(&createStreamReq); tFreeSCMCreateStreamReq(&createStreamReq);
tFreeStreamObj(&streamObj); tFreeStreamObj(&streamObj);

122
source/dnode/mnode/impl/src/mndSync.c
View File

@ -78,35 +78,38 @@ int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta
SSdbRaw *pRaw = pMsg->pCont; SSdbRaw *pRaw = pMsg->pCont;
int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw);
pMgmt->errCode = pMeta->code;
mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64
" role:%s raw:%p", " role:%s raw:%p sec:%d seq:%" PRId64,
transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state),
pRaw); pRaw, pMgmt->transSec, pMgmt->transSeq);
if (pMgmt->errCode == 0) { if (pMeta->code == 0) {
sdbWriteWithoutFree(pMnode->pSdb, pRaw); sdbWriteWithoutFree(pMnode->pSdb, pRaw);
sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex);
} }
taosWLockLatch(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
pMgmt->errCode = pMeta->code;
if (transId <= 0) { if (transId <= 0) {
taosWUnLockLatch(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
mError("trans:%d, invalid commit msg", transId); mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq);
} else if (transId == pMgmt->transId) { } else if (transId == pMgmt->transId) {
if (pMgmt->errCode != 0) { if (pMgmt->errCode != 0) {
mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode));
} else { } else {
mInfo("trans:%d, is proposed and post sem", transId); mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq);
} }
pMgmt->transId = 0; pMgmt->transId = 0;
pMgmt->transSec = 0;
pMgmt->transSeq = 0;
tsem_post(&pMgmt->syncSem); tsem_post(&pMgmt->syncSem);
taosWUnLockLatch(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
} else { } else {
taosWUnLockLatch(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
STrans *pTrans = mndAcquireTrans(pMnode, transId); STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans != NULL) { if (pTrans != NULL) {
mInfo("trans:%d, execute in mnode which not leader", transId); mInfo("trans:%d, execute in mnode which not leader or sync timeout", transId);
mndTransExecute(pMnode, pTrans); mndTransExecute(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans);
// sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA);
@ -198,18 +201,20 @@ int32_t mndSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int3
} }
static void mndBecomeFollower(const SSyncFSM *pFsm) { static void mndBecomeFollower(const SSyncFSM *pFsm) {
SMnode *pMnode = pFsm->data; SMnode *pMnode = pFsm->data;
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
mInfo("vgId:1, become follower"); mInfo("vgId:1, become follower");
taosWLockLatch(&pMnode->syncMgmt.lock); taosThreadMutexLock(&pMgmt->lock);
if (pMnode->syncMgmt.transId != 0) { if (pMgmt->transId != 0) {
mInfo("vgId:1, become follower and post sem, trans:%d, failed to propose since not leader", mInfo("vgId:1, become follower and post sem, trans:%d, failed to propose since not leader", pMgmt->transId);
pMnode->syncMgmt.transId); pMgmt->transId = 0;
pMnode->syncMgmt.transId = 0; pMgmt->transSec = 0;
pMnode->syncMgmt.errCode = TSDB_CODE_SYN_NOT_LEADER; pMgmt->transSeq = 0;
tsem_post(&pMnode->syncMgmt.syncSem); pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER;
tsem_post(&pMgmt->syncSem);
} }
taosWUnLockLatch(&pMnode->syncMgmt.lock); taosThreadMutexUnlock(&pMgmt->lock);
} }
static void mndBecomeLeader(const SSyncFSM *pFsm) { static void mndBecomeLeader(const SSyncFSM *pFsm) {
@ -265,8 +270,10 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) {
int32_t mndInitSync(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
taosInitRWLatch(&pMgmt->lock); taosThreadMutexInit(&pMgmt->lock, NULL);
pMgmt->transId = 0; pMgmt->transId = 0;
pMgmt->transSec = 0;
pMgmt->transSeq = 0;
SSyncInfo syncInfo = { SSyncInfo syncInfo = {
.snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT, .snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT,
@ -313,12 +320,38 @@ void mndCleanupSync(SMnode *pMnode) {
mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync);
tsem_destroy(&pMgmt->syncSem); tsem_destroy(&pMgmt->syncSem);
taosThreadMutexDestroy(&pMgmt->lock);
memset(pMgmt, 0, sizeof(SSyncMgmt)); memset(pMgmt, 0, sizeof(SSyncMgmt));
} }
void mndSyncCheckTimeout(SMnode *pMnode) {
mTrace("check sync timeout");
SSyncMgmt *pMgmt = &pMnode->syncMgmt;
taosThreadMutexLock(&pMgmt->lock);
if (pMgmt->transId != 0) {
int32_t curSec = taosGetTimestampSec();
int32_t delta = curSec - pMgmt->transSec;
if (delta > MNODE_TIMEOUT_SEC) {
mError("trans:%d, failed to propose since timeout, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId,
pMgmt->transSec, curSec, delta, pMgmt->transSeq);
pMgmt->transId = 0;
pMgmt->transSec = 0;
pMgmt->transSeq = 0;
terrno = TSDB_CODE_SYN_TIMEOUT;
pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT;
tsem_post(&pMgmt->syncSem);
} else {
mDebug("trans:%d, waiting for sync confirm, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId,
pMgmt->transSec, curSec, curSec - pMgmt->transSec, pMgmt->transSeq);
}
} else {
// mTrace("check sync timeout msg, no trans waiting for confirm");
}
taosThreadMutexUnlock(&pMgmt->lock);
}
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSyncMgmt *pMgmt = &pMnode->syncMgmt;
pMgmt->errCode = 0;
SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)};
if (req.contLen <= 0) return -1; if (req.contLen <= 0) return -1;
@ -327,35 +360,42 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
if (req.pCont == NULL) return -1; if (req.pCont == NULL) return -1;
memcpy(req.pCont, pRaw, req.contLen); memcpy(req.pCont, pRaw, req.contLen);
taosWLockLatch(&pMgmt->lock); taosThreadMutexLock(&pMgmt->lock);
pMgmt->errCode = 0;
if (pMgmt->transId != 0) { if (pMgmt->transId != 0) {
mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId);
taosWUnLockLatch(&pMgmt->lock); taosThreadMutexUnlock(&pMgmt->lock);
terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED;
return -1; return terrno;
} }
mInfo("trans:%d, will be proposed", transId); mInfo("trans:%d, will be proposed", transId);
pMgmt->transId = transId; pMgmt->transId = transId;
taosWUnLockLatch(&pMgmt->lock); pMgmt->transSec = taosGetTimestampSec();
int32_t code = syncPropose(pMgmt->sync, &req, false); int64_t seq = 0;
int32_t code = syncPropose(pMgmt->sync, &req, false, &seq);
if (code == 0) { if (code == 0) {
mInfo("trans:%d, is proposing and wait sem", pMgmt->transId); mInfo("trans:%d, is proposing and wait sem, seq:%" PRId64, transId, seq);
pMgmt->transSeq = seq;
taosThreadMutexUnlock(&pMgmt->lock);
tsem_wait(&pMgmt->syncSem); tsem_wait(&pMgmt->syncSem);
} else if (code > 0) { } else if (code > 0) {
mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId);
taosWLockLatch(&pMgmt->lock);
pMgmt->transId = 0; pMgmt->transId = 0;
taosWUnLockLatch(&pMgmt->lock); pMgmt->transSec = 0;
pMgmt->transSeq = 0;
taosThreadMutexUnlock(&pMgmt->lock);
sdbWriteWithoutFree(pMnode->pSdb, pRaw); sdbWriteWithoutFree(pMnode->pSdb, pRaw);
sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID); sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID);
code = 0; code = 0;
} else { } else {
mError("trans:%d, failed to proposed since %s", transId, terrstr()); mError("trans:%d, failed to proposed since %s", transId, terrstr());
taosWLockLatch(&pMgmt->lock);
pMgmt->transId = 0; pMgmt->transId = 0;
taosWUnLockLatch(&pMgmt->lock); pMgmt->transSec = 0;
pMgmt->transSeq = 0;
taosThreadMutexUnlock(&pMgmt->lock);
if (terrno == 0) { if (terrno == 0) {
terrno = TSDB_CODE_APP_ERROR; terrno = TSDB_CODE_APP_ERROR;
} }
@ -369,7 +409,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) {
} }
terrno = pMgmt->errCode; terrno = pMgmt->errCode;
return pMgmt->errCode; return terrno;
} }
void mndSyncStart(SMnode *pMnode) { void mndSyncStart(SMnode *pMnode) {
@ -382,13 +422,17 @@ void mndSyncStart(SMnode *pMnode) {
} }
void mndSyncStop(SMnode *pMnode) { void mndSyncStop(SMnode *pMnode) {
taosWLockLatch(&pMnode->syncMgmt.lock); SSyncMgmt *pMgmt = &pMnode->syncMgmt;
if (pMnode->syncMgmt.transId != 0) {
mInfo("vgId:1, is stopped and post sem, trans:%d", pMnode->syncMgmt.transId); taosThreadMutexLock(&pMgmt->lock);
pMnode->syncMgmt.transId = 0; if (pMgmt->transId != 0) {
tsem_post(&pMnode->syncMgmt.syncSem); mInfo("vgId:1, is stopped and post sem, trans:%d", pMgmt->transId);
pMgmt->transId = 0;
pMgmt->transSec = 0;
pMgmt->errCode = TSDB_CODE_APP_IS_STOPPING;
tsem_post(&pMgmt->syncSem);
} }
taosWUnLockLatch(&pMnode->syncMgmt.lock); taosThreadMutexUnlock(&pMgmt->lock);
} }
bool mndIsLeader(SMnode *pMnode) { bool mndIsLeader(SMnode *pMnode) {

22
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -838,7 +838,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
return conflict; return conflict;
} }
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) {
if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) {
if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) { if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT; terrno = TSDB_CODE_MND_TRANS_CONFLICT;
@ -853,6 +853,14 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
return -1; return -1;
} }
return 0;
}
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
if (mndTrancCheckConflict(pMnode, pTrans) != 0) {
return -1;
}
if (taosArrayGetSize(pTrans->commitActions) <= 0) { if (taosArrayGetSize(pTrans->commitActions) <= 0) {
terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL; terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL;
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -919,7 +927,8 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
} }
} else { } else {
if (pTrans->stage == TRN_STAGE_REDO_ACTION) { if (pTrans->stage == TRN_STAGE_REDO_ACTION) {
if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING) { if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING ||
code == TSDB_CODE_SYN_PROPOSE_NOT_READY) {
if (pTrans->failedTimes > 60) sendRsp = true; if (pTrans->failedTimes > 60) sendRsp = true;
} else { } else {
if (pTrans->failedTimes > 6) sendRsp = true; if (pTrans->failedTimes > 6) sendRsp = true;
@ -1027,6 +1036,7 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) {
if (pAction != NULL) { if (pAction != NULL) {
pAction->msgReceived = 1; pAction->msgReceived = 1;
pAction->errCode = pRsp->code; pAction->errCode = pRsp->code;
pTrans->lastErrorNo = pRsp->code;
} }
mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage), mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage),
@ -1238,7 +1248,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
if (numOfActions == 0) return code; if (numOfActions == 0) return code;
if (pTrans->redoActionPos >= numOfActions) return code; if (pTrans->redoActionPos >= numOfActions) return code;
mInfo("trans:%d, execute %d actions serial", pTrans->id, numOfActions); mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->redoActionPos);
for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos);
@ -1289,13 +1299,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
} else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mInfo("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); mInfo("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id);
break; break;
} else if (code == pAction->retryCode) { } else if (code == pAction->retryCode || code == TSDB_CODE_SYN_PROPOSE_NOT_READY ||
code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_SYN_NOT_LEADER) {
mInfo("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); mInfo("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code);
pTrans->lastErrorNo = code;
taosMsleep(300); taosMsleep(300);
action--; action--;
continue; continue;
} else { } else {
terrno = code; terrno = code;
pTrans->lastErrorNo = code;
pTrans->code = code; pTrans->code = code;
mInfo("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, mInfo("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id,
mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes);
@ -1324,6 +1337,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) {
} }
if (mndCannotExecuteTransAction(pMnode)) return false; if (mndCannotExecuteTransAction(pMnode)) return false;
terrno = code;
if (code == 0) { if (code == 0) {
pTrans->code = 0; pTrans->code = 0;

10
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -1126,8 +1126,12 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
} }
if (!force) { if (!force) {
#if 1
{
#else
if (newVg.replica == 1) { if (newVg.replica == 1) {
mInfo("vgId:%d, will add 1 vnode, replca:1", pVgroup->vgId); #endif
mInfo("vgId:%d, will add 1 vnode, replca:%d", pVgroup->vgId, newVg.replica);
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
for (int32_t i = 0; i < newVg.replica - 1; ++i) { for (int32_t i = 0; i < newVg.replica - 1; ++i) {
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
@ -1155,6 +1159,9 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1;
} }
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
#if 1
}
#else
} else { // new replica == 3 } else { // new replica == 3
mInfo("vgId:%d, will add 1 vnode, replca:3", pVgroup->vgId); mInfo("vgId:%d, will add 1 vnode, replca:3", pVgroup->vgId);
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;
@ -1181,6 +1188,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb,
if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1;
if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1;
} }
#endif
} else { } else {
mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId); mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId);
if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1;

1
source/dnode/vnode/inc/vnode.h
View File

@ -54,6 +54,7 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs);
void vnodeDestroy(const char *path, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs);
SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
void vnodePreClose(SVnode *pVnode); void vnodePreClose(SVnode *pVnode);
void vnodeSyncCheckTimeout(SVnode* pVnode);
void vnodeClose(SVnode *pVnode); void vnodeClose(SVnode *pVnode);
int32_t vnodeStart(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode);

14
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -87,17 +87,24 @@ typedef struct SCommitInfo SCommitInfo;
#define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA1_DIR "rsma1"
#define VNODE_RSMA2_DIR "rsma2" #define VNODE_RSMA2_DIR "rsma2"
#define VND_INFO_FNAME "vnode.json"
// vnd.h // vnd.h
void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMalloc(SVBufPool* pPool, int size);
void vnodeBufPoolFree(SVBufPool* pPool, void* p); void vnodeBufPoolFree(SVBufPool* pPool, void* p);
void vnodeBufPoolRef(SVBufPool* pPool); void vnodeBufPoolRef(SVBufPool* pPool);
void vnodeBufPoolUnRef(SVBufPool* pPool); void vnodeBufPoolUnRef(SVBufPool* pPool);
int vnodeDecodeInfo(uint8_t* pData, SVnodeInfo* pInfo);
// meta // meta
typedef struct SMCtbCursor SMCtbCursor; typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMStbCursor SMStbCursor; typedef struct SMStbCursor SMStbCursor;
typedef struct STbUidStore STbUidStore; typedef struct STbUidStore STbUidStore;
#define META_BEGIN_HEAP_BUFFERPOOL 0
#define META_BEGIN_HEAP_OS 1
#define META_BEGIN_HEAP_NIL 2
int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback);
int metaClose(SMeta* pMeta); int metaClose(SMeta* pMeta);
int metaBegin(SMeta* pMeta, int8_t fromSys); int metaBegin(SMeta* pMeta, int8_t fromSys);
@ -105,6 +112,7 @@ TXN* metaGetTxn(SMeta* pMeta);
int metaCommit(SMeta* pMeta, TXN* txn); int metaCommit(SMeta* pMeta, TXN* txn);
int metaFinishCommit(SMeta* pMeta, TXN* txn); int metaFinishCommit(SMeta* pMeta, TXN* txn);
int metaPrepareAsyncCommit(SMeta* pMeta); int metaPrepareAsyncCommit(SMeta* pMeta);
int metaAbort(SMeta* pMeta);
int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq);
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList);
@ -238,6 +246,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
// STsdbSnapWriter ======================================== // STsdbSnapWriter ========================================
int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter); int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData); int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter);
int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
// STqSnapshotReader == // STqSnapshotReader ==
int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader);
@ -344,7 +353,12 @@ struct SVnode {
bool blocked; bool blocked;
bool restored; bool restored;
tsem_t syncSem; tsem_t syncSem;
int32_t blockSec;
int64_t blockSeq;
SQHandle* pQuery; SQHandle* pQuery;
#if 0
SRpcHandleInfo blockInfo;
#endif
}; };
#define TD_VID(PVNODE) ((PVNODE)->config.vgId) #define TD_VID(PVNODE) ((PVNODE)->config.vgId)

12
source/dnode/vnode/src/meta/metaCommit.c
View File

@ -19,19 +19,21 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBuf
static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); }
// begin a meta txn // begin a meta txn
int metaBegin(SMeta *pMeta, int8_t fromSys) { int metaBegin(SMeta *pMeta, int8_t heap) {
void *(*xMalloc)(void *, size_t); void *(*xMalloc)(void *, size_t) = NULL;
void (*xFree)(void *, void *); void (*xFree)(void *, void *) = NULL;
void *xArg = NULL; void *xArg = NULL;
if (fromSys) { // default heap to META_BEGIN_HEAP_NIL
if (heap == META_BEGIN_HEAP_OS) {
xMalloc = tdbDefaultMalloc; xMalloc = tdbDefaultMalloc;
xFree = tdbDefaultFree; xFree = tdbDefaultFree;
} else { } else if (heap == META_BEGIN_HEAP_BUFFERPOOL) {
xMalloc = metaMalloc; xMalloc = metaMalloc;
xFree = metaFree; xFree = metaFree;
xArg = pMeta->pVnode->inUse; xArg = pMeta->pVnode->inUse;
} }
if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) {
return -1; return -1;
} }

5
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -145,7 +145,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
metaBegin(pMeta, 1); metaBegin(pMeta, META_BEGIN_HEAP_NIL);
*ppWriter = pWriter; *ppWriter = pWriter;
return code; return code;
@ -161,7 +161,8 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) {
SMetaSnapWriter* pWriter = *ppWriter; SMetaSnapWriter* pWriter = *ppWriter;
if (rollback) { if (rollback) {
ASSERT(0); code = metaAbort(pWriter->pMeta);
if (code) goto _err;
} else { } else {
code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn); code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn);
if (code) goto _err; if (code) goto _err;

2
source/dnode/vnode/src/tq/tq.c
View File

@ -738,7 +738,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL
code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey); code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey);
if (code != 0) { if (code != 0) {
tqError("cannot process tq delete req %s, since no such offset", pReq->subKey); tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey);
} }
if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) {

1
source/dnode/vnode/src/tq/tqRead.c
View File

@ -533,6 +533,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
pBlock->info.id.uid = pReader->msgIter.uid; pBlock->info.id.uid = pReader->msgIter.uid;
pBlock->info.rows = pReader->msgIter.numOfRows; pBlock->info.rows = pReader->msgIter.numOfRows;
pBlock->info.version = pReader->pMsg->version; pBlock->info.version = pReader->pMsg->version;
pBlock->info.dataLoad = 1;
while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) {
tdSTSRowIterReset(&iter, row); tdSTSRowIterReset(&iter, row);

4
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1143,6 +1143,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
i += 1; i += 1;
} }
pResBlock->info.dataLoad = 1;
pResBlock->info.rows = dumpedRows; pResBlock->info.rows = dumpedRows;
pDumpInfo->rowIndex += step * dumpedRows; pDumpInfo->rowIndex += step * dumpedRows;
@ -2538,6 +2539,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
_end: _end:
pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
pResBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
setComposedBlockFlag(pReader, true); setComposedBlockFlag(pReader, true);
@ -3622,6 +3624,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
i += 1; i += 1;
} }
pBlock->info.dataLoad = 1;
pBlock->info.rows += 1; pBlock->info.rows += 1;
pScanInfo->lastKey = pTSRow->ts; pScanInfo->lastKey = pTSRow->ts;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -3669,6 +3672,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
i += 1; i += 1;
} }
pResBlock->info.dataLoad = 1;
pResBlock->info.rows += 1; pResBlock->info.rows += 1;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

35
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -1376,27 +1376,34 @@ _exit:
return code; return code;
} }
int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter) {
int32_t code = 0;
if (pWriter->dWriter.pWriter) {
code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _exit;
}
code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _exit;
code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs);
if (code) goto _exit;
_exit:
if (code) {
tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
}
return code;
}
int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
int32_t code = 0; int32_t code = 0;
STsdbSnapWriter* pWriter = *ppWriter; STsdbSnapWriter* pWriter = *ppWriter;
STsdb* pTsdb = pWriter->pTsdb; STsdb* pTsdb = pWriter->pTsdb;
if (rollback) { if (rollback) {
ASSERT(0); tsdbRollbackCommit(pWriter->pTsdb);
// code = tsdbFSRollback(pWriter->pTsdb->pFS);
// if (code) goto _err;
} else { } else {
if (pWriter->dWriter.pWriter) {
code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _err;
}
code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _err;
code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err;
// lock // lock
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);

8
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -115,7 +115,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
@ -135,9 +134,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
} }
// add tsdb page size config
if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1;
return 0; return 0;
} }
@ -256,7 +252,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
} }
tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code);
if (code < 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; if (code < 0 || pCfg->tsdbPageSize < TSDB_MIN_PAGESIZE_PER_VNODE * 1024) {
pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024;
}
return 0; return 0;
} }

6
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -16,11 +16,9 @@
#include "vnd.h" #include "vnd.h"
#include "vnodeInt.h" #include "vnodeInt.h"
#define VND_INFO_FNAME "vnode.json"
#define VND_INFO_FNAME_TMP "vnode_tmp.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json"
static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
static int vnodeCommitImpl(SCommitInfo *pInfo); static int vnodeCommitImpl(SCommitInfo *pInfo);
int vnodeBegin(SVnode *pVnode) { int vnodeBegin(SVnode *pVnode) {
@ -40,7 +38,7 @@ int vnodeBegin(SVnode *pVnode) {
pVnode->state.commitID++; pVnode->state.commitID++;
// begin meta // begin meta
if (metaBegin(pVnode->pMeta, 0) < 0) { if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) {
vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno));
return -1; return -1;
} }
@ -407,7 +405,7 @@ _err:
return -1; return -1;
} }
static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) { int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
SJson *pJson = NULL; SJson *pJson = NULL;
pJson = tjsonParse(pData); pJson = tjsonParse(pData);

165
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -21,6 +21,8 @@ struct SVSnapReader {
int64_t sver; int64_t sver;
int64_t ever; int64_t ever;
int64_t index; int64_t index;
// config
int8_t cfgDone;
// meta // meta
int8_t metaDone; int8_t metaDone;
SMetaSnapReader *pMetaReader; SMetaSnapReader *pMetaReader;
@ -88,6 +90,53 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader) {
int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) {
int32_t code = 0; int32_t code = 0;
// CONFIG ==============
// FIXME: if commit multiple times and the config changed?
if (!pReader->cfgDone) {
char fName[TSDB_FILENAME_LEN];
if (pReader->pVnode->pTfs) {
snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pReader->pVnode->pTfs), TD_DIRSEP,
pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME);
} else {
snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s", pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME);
}
TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ);
if (NULL == pFile) {
code = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
int64_t size;
if (taosFStatFile(pFile, &size, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosCloseFile(&pFile);
goto _err;
}
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
taosCloseFile(&pFile);
goto _err;
}
((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG;
((SSnapDataHdr *)(*ppData))->size = size + 1;
((SSnapDataHdr *)(*ppData))->data[size] = '\0';
if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(*ppData);
taosCloseFile(&pFile);
goto _err;
}
taosCloseFile(&pFile);
pReader->cfgDone = 1;
goto _exit;
}
// META ============== // META ==============
if (!pReader->metaDone) { if (!pReader->metaDone) {
// open reader if not // open reader if not
@ -230,6 +279,8 @@ struct SVSnapWriter {
int64_t ever; int64_t ever;
int64_t commitID; int64_t commitID;
int64_t index; int64_t index;
// config
SVnodeInfo info;
// meta // meta
SMetaSnapWriter *pMetaSnapWriter; SMetaSnapWriter *pMetaSnapWriter;
// tsdb // tsdb
@ -248,6 +299,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
int32_t code = 0; int32_t code = 0;
SVSnapWriter *pWriter = NULL; SVSnapWriter *pWriter = NULL;
// commit memory data
vnodeAsyncCommit(pVnode);
tsem_wait(&pVnode->canCommit);
// alloc // alloc
pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
if (pWriter == NULL) { if (pWriter == NULL) {
@ -258,16 +313,8 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
pWriter->sver = sver; pWriter->sver = sver;
pWriter->ever = ever; pWriter->ever = ever;
// commit it
code = vnodeSyncCommit(pVnode);
if (code) {
taosMemoryFree(pWriter);
goto _err;
}
// inc commit ID // inc commit ID
pVnode->state.commitID++; pWriter->commitID = ++pVnode->state.commitID;
pWriter->commitID = pVnode->state.commitID;
vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode), vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
sver, ever, pWriter->commitID); sver, ever, pWriter->commitID);
@ -284,53 +331,89 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
int32_t code = 0; int32_t code = 0;
SVnode *pVnode = pWriter->pVnode; SVnode *pVnode = pWriter->pVnode;
// prepare
if (pWriter->pTsdbSnapWriter) {
tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter);
}
// commit json
if (!rollback) {
pVnode->config = pWriter->info.config;
pVnode->state = (SVState){.committed = pWriter->info.state.committed,
.applied = pWriter->info.state.committed,
.commitID = pWriter->commitID,
.commitTerm = pWriter->info.state.commitTerm,
.applyTerm = pWriter->info.state.commitTerm};
pVnode->statis = pWriter->info.statis;
char dir[TSDB_FILENAME_LEN] = {0};
if (pWriter->pVnode->pTfs) {
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
} else {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path);
}
vnodeCommitInfo(dir, &pWriter->info);
} else {
vnodeRollback(pWriter->pVnode);
}
// commit/rollback sub-system
if (pWriter->pMetaSnapWriter) { if (pWriter->pMetaSnapWriter) {
code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback); code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback);
if (code) goto _err; if (code) goto _exit;
} }
if (pWriter->pTsdbSnapWriter) { if (pWriter->pTsdbSnapWriter) {
code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback);
if (code) goto _err; if (code) goto _exit;
} }
if (pWriter->pRsmaSnapWriter) { if (pWriter->pRsmaSnapWriter) {
code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback);
if (code) goto _err; if (code) goto _exit;
} }
if (!rollback) { vnodeBegin(pVnode);
SVnodeInfo info = {0};
char dir[TSDB_FILENAME_LEN];
pVnode->state.committed = pWriter->ever; _exit:
pVnode->state.applied = pWriter->ever; if (code) {
pVnode->state.applyTerm = pSnapshot->lastApplyTerm; vError("vgId:%d, vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
pVnode->state.commitTerm = pSnapshot->lastApplyTerm;
info.config = pVnode->config;
info.state.committed = pVnode->state.applied;
info.state.commitTerm = pVnode->state.applyTerm;
info.state.commitID = pVnode->state.commitID;
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
code = vnodeSaveInfo(dir, &info);
if (code) goto _err;
code = vnodeCommitInfo(dir, &info);
if (code) goto _err;
vnodeBegin(pVnode);
} else { } else {
ASSERT(0); vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback);
taosMemoryFree(pWriter);
}
tsem_post(&pVnode->canCommit);
return code;
}
static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
int32_t code = 0;
SSnapDataHdr *pHdr = (SSnapDataHdr *)pData;
// decode info
if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) {
code = TSDB_CODE_INVALID_MSG;
goto _exit;
}
// change some value
pWriter->info.state.commitID = pWriter->commitID;
// modify info as needed
char dir[TSDB_FILENAME_LEN] = {0};
if (pWriter->pVnode->pTfs) {
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pWriter->pVnode->pTfs), TD_DIRSEP,
pWriter->pVnode->path);
} else {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path);
}
if (vnodeSaveInfo(dir, &pWriter->info) < 0) {
code = terrno;
goto _exit;
} }
_exit: _exit:
vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback);
taosMemoryFree(pWriter);
return code;
_err:
vError("vgId:%d, vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code));
return code; return code;
} }
@ -347,6 +430,10 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
pHdr->type, nData); pHdr->type, nData);
switch (pHdr->type) { switch (pHdr->type) {
case SNAP_DATA_CFG: {
code = vnodeSnapWriteInfo(pWriter, pData, nData);
if (code) goto _err;
} break;
case SNAP_DATA_META: { case SNAP_DATA_META: {
// meta // meta
if (pWriter->pMetaSnapWriter == NULL) { if (pWriter->pMetaSnapWriter == NULL) {

4
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -317,11 +317,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
// commit if need // commit if need
if (vnodeShouldCommit(pVnode)) { if (vnodeShouldCommit(pVnode)) {
vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version);
#if 0
vnodeSyncCommit(pVnode);
#else
vnodeAsyncCommit(pVnode); vnodeAsyncCommit(pVnode);
#endif
// start a new one // start a new one
if (vnodeBegin(pVnode) < 0) { if (vnodeBegin(pVnode) < 0) {

69
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -22,32 +22,21 @@ static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; }
static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
vGTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); vGTrace("vgId:%d, msg:%p wait block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg,
TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq);
tsem_wait(&pVnode->syncSem); tsem_wait(&pVnode->syncSem);
} }
static inline void vnodeWaitBlockMsgOld(SVnode *pVnode, const SRpcMsg *pMsg) {
if (vnodeIsMsgBlock(pMsg->msgType)) {
const STraceId *trace = &pMsg->info.traceId;
taosThreadMutexLock(&pVnode->lock);
if (!pVnode->blocked) {
vGTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType));
pVnode->blocked = true;
taosThreadMutexUnlock(&pVnode->lock);
tsem_wait(&pVnode->syncSem);
} else {
taosThreadMutexUnlock(&pVnode->lock);
}
}
}
static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
if (vnodeIsMsgBlock(pMsg->msgType)) { if (vnodeIsMsgBlock(pMsg->msgType)) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
taosThreadMutexLock(&pVnode->lock); taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) { if (pVnode->blocked) {
vGTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); vGTrace("vgId:%d, msg:%p post block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg,
TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq);
pVnode->blocked = false; pVnode->blocked = false;
pVnode->blockSec = 0;
pVnode->blockSeq = 0;
tsem_post(&pVnode->syncSem); tsem_post(&pVnode->syncSem);
} }
taosThreadMutexUnlock(&pVnode->lock); taosThreadMutexUnlock(&pVnode->lock);
@ -217,12 +206,19 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs)
#else #else
static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) {
int64_t seq = 0;
taosThreadMutexLock(&pVnode->lock); taosThreadMutexLock(&pVnode->lock);
int32_t code = syncPropose(pVnode->sync, pMsg, isWeak); int32_t code = syncPropose(pVnode->sync, pMsg, isWeak, &seq);
bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType));
if (wait) { if (wait) {
ASSERT(!pVnode->blocked); ASSERT(!pVnode->blocked);
pVnode->blocked = true; pVnode->blocked = true;
pVnode->blockSec = taosGetTimestampSec();
pVnode->blockSeq = seq;
#if 0
pVnode->blockInfo = pMsg->info;
#endif
} }
taosThreadMutexUnlock(&pVnode->lock); taosThreadMutexUnlock(&pVnode->lock);
@ -289,12 +285,15 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; if (taosGetQitem(qall, (void **)&pMsg) == 0) continue;
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg,
TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex);
if (vnodeIsMsgBlock(pMsg->msgType)) { if (vnodeIsMsgBlock(pMsg->msgType)) {
vTrace("vgId:%d, blocking msg obtained from apply-queue. index:%" PRId64 ", term: %" PRId64 ", type: %s", vgId, vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64
pMsg->info.conn.applyIndex, pMsg->info.conn.applyTerm, TMSG_INFO(pMsg->msgType)); ", blocking msg obtained sec:%d seq:%" PRId64,
vgId, pMsg, TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex, pVnode->blockSec,
pVnode->blockSeq);
} else {
vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg,
TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex);
} }
SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info};
@ -621,6 +620,32 @@ void vnodeSyncClose(SVnode *pVnode) {
syncStop(pVnode->sync); syncStop(pVnode->sync);
} }
void vnodeSyncCheckTimeout(SVnode *pVnode) {
vTrace("vgId:%d, check sync timeout msg", pVnode->config.vgId);
taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
int32_t curSec = taosGetTimestampSec();
int32_t delta = curSec - pVnode->blockSec;
if (delta > VNODE_TIMEOUT_SEC) {
vError("vgId:%d, failed to propose since timeout and post block, start:%d cur:%d delta:%d seq:%" PRId64,
pVnode->config.vgId, pVnode->blockSec, curSec, delta, pVnode->blockSeq);
if (syncSendTimeoutRsp(pVnode->sync, pVnode->blockSeq) != 0) {
#if 0
SRpcMsg rpcMsg = {.code = TSDB_CODE_SYN_TIMEOUT, .info = pVnode->blockInfo};
vError("send timeout response since its applyed, seq:%" PRId64 " handle:%p ahandle:%p", pVnode->blockSeq,
rpcMsg.info.handle, rpcMsg.info.ahandle);
rpcSendResponse(&rpcMsg);
#endif
}
pVnode->blocked = false;
pVnode->blockSec = 0;
pVnode->blockSeq = 0;
tsem_post(&pVnode->syncSem);
}
}
taosThreadMutexUnlock(&pVnode->lock);
}
bool vnodeIsRoleLeader(SVnode *pVnode) { bool vnodeIsRoleLeader(SVnode *pVnode) {
SSyncState state = syncGetState(pVnode->sync); SSyncState state = syncGetState(pVnode->sync);
return state.state == TAOS_SYNC_STATE_LEADER; return state.state == TAOS_SYNC_STATE_LEADER;

2
source/libs/catalog/src/ctgCache.c
View File

@ -1728,7 +1728,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
} }
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, ctgDebug("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash,
dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_JRET(TSDB_CODE_APP_ERROR); CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
} }

1
source/libs/executor/src/exchangeoperator.c
View File

@ -510,6 +510,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
blockDataEnsureCapacity(pRes, pBlock->info.rows); blockDataEnsureCapacity(pRes, pBlock->info.rows);
// data from mnode // data from mnode
pRes->info.dataLoad = 1;
pRes->info.rows = pBlock->info.rows; pRes->info.rows = pBlock->info.rows;
relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); relocateColumnData(pRes, pColList, pBlock->pDataBlock, false);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);

4
source/libs/executor/src/executorimpl.c
View File

@ -1080,7 +1080,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.id.groupId); pBlock->info.id.groupId);
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
return 0; return 0;
} }
@ -2546,6 +2546,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
pBlock->info.rows += pRow->numOfRows; pBlock->info.rows += pRow->numOfRows;
releaseOutputBuf(pState, &key, pRow); releaseOutputBuf(pState, &key, pRow);
} }
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2635,6 +2636,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
} }
} }
pBlock->info.dataLoad = 1;
pBlock->info.rows += pRow->numOfRows; pBlock->info.rows += pRow->numOfRows;
// saveSessionDiscBuf(pState, pKey, pVal, size); // saveSessionDiscBuf(pState, pKey, pVal, size);
releaseOutputBuf(pState, NULL, pRow); releaseOutputBuf(pState, NULL, pRow);

3
source/libs/executor/src/groupoperator.c
View File

@ -698,6 +698,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
pInfo->pageIndex += 1; pInfo->pageIndex += 1;
releaseBufPage(pInfo->pBuf, page); releaseBufPage(pInfo->pBuf, page);
pInfo->binfo.pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->binfo.pRes, 0); blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId;
@ -960,6 +961,8 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
} }
taosArrayDestroy(pParInfo->rowIds); taosArrayDestroy(pParInfo->rowIds);
pParInfo->rowIds = NULL; pParInfo->rowIds = NULL;
pDest->info.dataLoad = 1;
blockDataUpdateTsWindow(pDest, pInfo->tsColIndex); blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
pDest->info.id.groupId = pParInfo->groupId; pDest->info.id.groupId = pParInfo->groupId;
pOperator->resultInfo.totalRows += pDest->info.rows; pOperator->resultInfo.totalRows += pDest->info.rows;

9
source/libs/executor/src/joinoperator.c
View File

@ -87,11 +87,11 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
} }
int32_t numOfCols = 0; int32_t numOfCols = 0;
SSDataBlock* pResBlock = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); pInfo->pRes = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols); SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols);
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
pInfo->pRes = pResBlock;
setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.pExprInfo = pExprInfo;
@ -401,6 +401,7 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes)
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
pRes->info.rows = nrows; pRes->info.rows = nrows;
pRes->info.dataLoad = 1;
if (pRes->info.rows >= pOperator->resultInfo.threshold) { if (pRes->info.rows >= pOperator->resultInfo.threshold) {
break; break;
} }
@ -412,7 +413,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) {
SSDataBlock* pRes = pJoinInfo->pRes; SSDataBlock* pRes = pJoinInfo->pRes;
blockDataCleanup(pRes); blockDataCleanup(pRes);
blockDataEnsureCapacity(pRes, 4096);
while (true) { while (true) {
int32_t numOfRowsBefore = pRes->info.rows; int32_t numOfRowsBefore = pRes->info.rows;
doMergeJoinImpl(pOperator, pRes); doMergeJoinImpl(pOperator, pRes);

1
source/libs/executor/src/projectoperator.c
View File

@ -654,6 +654,7 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList) { int32_t numOfOutput, SArray* pPseudoList) {
setPseudoOutputColInfo(pResult, pCtx, pPseudoList); setPseudoOutputColInfo(pResult, pCtx, pPseudoList);
pResult->info.dataLoad = 1;
if (pSrcBlock == NULL) { if (pSrcBlock == NULL) {
for (int32_t k = 0; k < numOfOutput; ++k) { for (int32_t k = 0; k < numOfOutput; ++k) {

15
source/libs/executor/src/scanoperator.c
View File

@ -110,9 +110,9 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
if (order == TSDB_ORDER_ASC) { if (order == TSDB_ORDER_ASC) {
w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey);
assert(w.ekey >= pBlockInfo->window.skey); ASSERT(w.ekey >= pBlockInfo->window.skey);
if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) { if (w.ekey < pBlockInfo->window.ekey) {
return true; return true;
} }
@ -122,16 +122,16 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
break; break;
} }
assert(w.ekey > pBlockInfo->window.ekey); ASSERT(w.ekey > pBlockInfo->window.ekey);
if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) { if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) {
return true; return true;
} }
} }
} else { } else {
w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey);
assert(w.skey <= pBlockInfo->window.ekey); ASSERT(w.skey <= pBlockInfo->window.ekey);
if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) { if (w.skey > pBlockInfo->window.skey) {
return true; return true;
} }
@ -1342,6 +1342,7 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock,
} }
pDestBlock->info.type = STREAM_CLEAR; pDestBlock->info.type = STREAM_CLEAR;
pDestBlock->info.version = pSrcBlock->info.version; pDestBlock->info.version = pSrcBlock->info.version;
pDestBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pDestBlock, 0); blockDataUpdateTsWindow(pDestBlock, 0);
return code; return code;
} }
@ -1450,6 +1451,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
} }
if (out && pInfo->pUpdateDataRes->info.rows > 0) { if (out && pInfo->pUpdateDataRes->info.rows > 0) {
pInfo->pUpdateDataRes->info.version = pBlock->info.version; pInfo->pUpdateDataRes->info.version = pBlock->info.version;
pInfo->pUpdateDataRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0); blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0);
pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR; pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR;
} }
@ -1512,6 +1514,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
} }
pInfo->pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
blockDataFreeRes((SSDataBlock*)pBlock); blockDataFreeRes((SSDataBlock*)pBlock);
@ -1793,6 +1796,7 @@ FETCH_NEXT_BLOCK:
// TODO move into scan // TODO move into scan
pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.skey = INT64_MIN;
pBlock->info.calWin.ekey = INT64_MAX; pBlock->info.calWin.ekey = INT64_MAX;
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
switch (pBlock->info.type) { switch (pBlock->info.type) {
case STREAM_NORMAL: case STREAM_NORMAL:
@ -1970,6 +1974,7 @@ FETCH_NEXT_BLOCK:
} }
doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
pInfo->pRes->info.dataLoad = 1;
blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) {

2
source/libs/executor/src/sortoperator.c
View File

@ -105,6 +105,7 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
} }
} }
pBlock->info.dataLoad = 1;
pBlock->info.rows += 1; pBlock->info.rows += 1;
} }
@ -698,6 +699,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
pInfo->limitInfo.numOfOutputRows += p->info.rows; pInfo->limitInfo.numOfOutputRows += p->info.rows;
pDataBlock->info.rows = p->info.rows; pDataBlock->info.rows = p->info.rows;
pDataBlock->info.id.groupId = pInfo->groupId; pDataBlock->info.id.groupId = pInfo->groupId;
pDataBlock->info.dataLoad = 1;
} }
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId,

5
source/libs/executor/src/timewindowoperator.c
View File

@ -1037,9 +1037,10 @@ SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SRe
int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) { int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) {
TSKEY* tsCols = NULL; TSKEY* tsCols = NULL;
if (pBlock->pDataBlock != NULL) { if (pBlock->pDataBlock != NULL && pBlock->info.dataLoad == 1) {
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData; tsCols = (int64_t*)pColDataInfo->pData;
ASSERT(tsCols[0] != 0);
// no data in primary ts // no data in primary ts
if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) { if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) {
@ -1083,8 +1084,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
// the pDataBlock are always the same one, no need to call this again // the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true); setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true);
blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
} }

1
source/libs/executor/test/executorTests.cpp
View File

@ -166,6 +166,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
pInfo->current += 1; pInfo->current += 1;
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0); blockDataUpdateTsWindow(pBlock, 0);
return pBlock; return pBlock;
} }

2
source/libs/function/src/detail/tavgfunction.c
View File

@ -514,7 +514,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
numOfElem = pInput->numOfRows; numOfElem = pInput->numOfRows;
pAvgRes->count += pInput->numOfRows; pAvgRes->count += pInput->numOfRows;
bool simdAvailable = tsAVXEnable && tsSIMDEnable && (numOfRows > THRESHOLD_SIZE); bool simdAvailable = tsAVXEnable && tsSIMDBuiltins && (numOfRows > THRESHOLD_SIZE);
switch(type) { switch(type) {
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:

10
source/libs/function/src/detail/tminmax.c
View File

@ -369,7 +369,7 @@ static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start,
static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
bool signVal) { bool signVal) {
// AVX2 version to speedup the loop // AVX2 version to speedup the loop
if (tsAVX2Enable && tsSIMDEnable) { if (tsAVX2Enable && tsSIMDBuiltins) {
pBuf->v = i8VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); pBuf->v = i8VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
} else { } else {
if (!pBuf->assign) { if (!pBuf->assign) {
@ -403,7 +403,7 @@ static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SM
static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
bool signVal) { bool signVal) {
// AVX2 version to speedup the loop // AVX2 version to speedup the loop
if (tsAVX2Enable && tsSIMDEnable) { if (tsAVX2Enable && tsSIMDBuiltins) {
pBuf->v = i16VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); pBuf->v = i16VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
} else { } else {
if (!pBuf->assign) { if (!pBuf->assign) {
@ -437,7 +437,7 @@ static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, S
static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc,
bool signVal) { bool signVal) {
// AVX2 version to speedup the loop // AVX2 version to speedup the loop
if (tsAVX2Enable && tsSIMDEnable) { if (tsAVX2Enable && tsSIMDBuiltins) {
pBuf->v = i32VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); pBuf->v = i32VectorCmpAVX2(data, numOfRows, isMinFunc, signVal);
} else { } else {
if (!pBuf->assign) { if (!pBuf->assign) {
@ -500,7 +500,7 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo
float* val = (float*)&pBuf->v; float* val = (float*)&pBuf->v;
// AVX version to speedup the loop // AVX version to speedup the loop
if (tsAVXEnable && tsSIMDEnable) { if (tsAVXEnable && tsSIMDBuiltins) {
*val = floatVectorCmpAVX(pData, numOfRows, isMinFunc); *val = floatVectorCmpAVX(pData, numOfRows, isMinFunc);
} else { } else {
if (!pBuf->assign) { if (!pBuf->assign) {
@ -530,7 +530,7 @@ static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfR
double* val = (double*)&pBuf->v; double* val = (double*)&pBuf->v;
// AVX version to speedup the loop // AVX version to speedup the loop
if (tsAVXEnable && tsSIMDEnable) { if (tsAVXEnable && tsSIMDBuiltins) {
*val = (double)doubleVectorCmpAVX(pData, numOfRows, isMinFunc); *val = (double)doubleVectorCmpAVX(pData, numOfRows, isMinFunc);
} else { } else {
if (!pBuf->assign) { if (!pBuf->assign) {

10
source/libs/function/src/tpercentile.c
View File

@ -364,6 +364,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
// keep the pointer in memory // keep the pointer in memory
setBufPageDirty(pSlot->info.data, true);
releaseBufPage(pBucket->pBuffer, pSlot->info.data); releaseBufPage(pBucket->pBuffer, pSlot->info.data);
pSlot->info.data = NULL; pSlot->info.data = NULL;
} }
@ -497,15 +498,16 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
resetSlotInfo(pMemBucket); resetSlotInfo(pMemBucket);
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
SArray* list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); SArray* list = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId));
ASSERT(list != NULL && list->size > 0); ASSERT(list != NULL && list->size > 0);
for (int32_t f = 0; f < list->size; ++f) { for (int32_t f = 0; f < list->size; ++f) {
SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); int32_t *pageId = taosArrayGet(list, f);
SFilePage *pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo)); SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId);
tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
releaseBufPageInfo(pMemBucket->pBuffer, pgInfo); setBufPageDirty(pg, true);
releaseBufPage(pMemBucket->pBuffer, pg);
} }
return getPercentileImpl(pMemBucket, count - num, fraction); return getPercentileImpl(pMemBucket, count - num, fraction);

27
source/libs/function/src/tudf.c
View File

@ -837,9 +837,34 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
} }
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
output->info.rows = input->numOfRows; int32_t numOfRows = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
numOfRows = (input[i].numOfRows > numOfRows) ? input[i].numOfRows : numOfRows;
}
output->info.rows = numOfRows;
output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
if ((input+i)->numOfRows < numOfRows) {
SColumnInfoData* pColInfoData = (input+i)->columnData;
int32_t startRow = (input+i)->numOfRows;
int32_t expandRows = numOfRows - startRow;
colInfoDataEnsureCapacity(pColInfoData, numOfRows, false);
bool isNull = colDataIsNull_s(pColInfoData, (input+i)->numOfRows - 1);
if (isNull) {
colDataAppendNNULL(pColInfoData, startRow, expandRows);
} else {
char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1);
int32_t bytes = pColInfoData->info.bytes;
char* data = taosMemoryMalloc(bytes);
memcpy(data, src, bytes);
for (int j = 0; j < expandRows; ++j) {
colDataAppend(pColInfoData, startRow+j, data, false);
}
//colDataAppendNItems(pColInfoData, startRow, data, expandRows);
taosMemoryFree(data);
}
}
taosArrayPush(output->pDataBlock, (input + i)->columnData); taosArrayPush(output->pDataBlock, (input + i)->columnData);
if (IS_VAR_DATA_TYPE((input + i)->columnData->info.type)) { if (IS_VAR_DATA_TYPE((input + i)->columnData->info.type)) {

124
source/libs/function/src/udfd.c
View File

@ -28,39 +28,46 @@
#include "tmsg.h" #include "tmsg.h"
#include "trpc.h" #include "trpc.h"
#include "tmisce.h" #include "tmisce.h"
// clang-foramt on // clang-format on
typedef struct SUdfdContext { typedef struct SUdfdContext {
uv_loop_t * loop; uv_loop_t *loop;
uv_pipe_t ctrlPipe; uv_pipe_t ctrlPipe;
uv_signal_t intrSignal; uv_signal_t intrSignal;
char listenPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; char listenPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2];
uv_pipe_t listeningPipe; uv_pipe_t listeningPipe;
void * clientRpc; void *clientRpc;
SCorEpSet mgmtEp; SCorEpSet mgmtEp;
uv_mutex_t udfsMutex; uv_mutex_t udfsMutex;
SHashObj * udfsHash; SHashObj *udfsHash;
SArray* residentFuncs; SArray *residentFuncs;
bool printVersion; bool printVersion;
} SUdfdContext; } SUdfdContext;
SUdfdContext global; SUdfdContext global;
struct SUdfdUvConn;
struct SUvUdfWork;
typedef struct SUdfdUvConn { typedef struct SUdfdUvConn {
uv_stream_t *client; uv_stream_t *client;
char * inputBuf; char *inputBuf;
int32_t inputLen; int32_t inputLen;
int32_t inputCap; int32_t inputCap;
int32_t inputTotal; int32_t inputTotal;
struct SUvUdfWork *pWorkList; // head of work list
} SUdfdUvConn; } SUdfdUvConn;
typedef struct SUvUdfWork { typedef struct SUvUdfWork {
uv_stream_t *client; SUdfdUvConn *conn;
uv_buf_t input; uv_buf_t input;
uv_buf_t output; uv_buf_t output;
struct SUvUdfWork *pWorkNext;
} SUvUdfWork; } SUvUdfWork;
typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState; typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState;
@ -70,7 +77,7 @@ typedef struct SUdf {
EUdfState state; EUdfState state;
uv_mutex_t lock; uv_mutex_t lock;
uv_cond_t condReady; uv_cond_t condReady;
bool resident; bool resident;
char name[TSDB_FUNC_NAME_LEN + 1]; char name[TSDB_FUNC_NAME_LEN + 1];
int8_t funcType; int8_t funcType;
@ -107,7 +114,7 @@ typedef enum EUdfdRpcReqRspType {
typedef struct SUdfdRpcSendRecvInfo { typedef struct SUdfdRpcSendRecvInfo {
EUdfdRpcReqRspType rpcType; EUdfdRpcReqRspType rpcType;
int32_t code; int32_t code;
void * param; void *param;
uv_sem_t resultSem; uv_sem_t resultSem;
} SUdfdRpcSendRecvInfo; } SUdfdRpcSendRecvInfo;
@ -178,7 +185,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName);
SUdfSetupRequest *setup = &request->setup; SUdfSetupRequest *setup = &request->setup;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SUdf * udf = NULL; SUdf *udf = NULL;
uv_mutex_lock(&global.udfsMutex); uv_mutex_lock(&global.udfsMutex);
SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName));
if (udfInHash) { if (udfInHash) {
@ -193,7 +200,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
uv_cond_init(&udfNew->condReady); uv_cond_init(&udfNew->condReady);
udf = udfNew; udf = udfNew;
SUdf** pUdf = &udf; SUdf **pUdf = &udf;
taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), pUdf, POINTER_BYTES); taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), pUdf, POINTER_BYTES);
uv_mutex_unlock(&global.udfsMutex); uv_mutex_unlock(&global.udfsMutex);
} }
@ -207,7 +214,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
} }
udf->resident = false; udf->resident = false;
for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) {
char* funcName = taosArrayGet(global.residentFuncs, i); char *funcName = taosArrayGet(global.residentFuncs, i);
if (strcmp(setup->udfName, funcName) == 0) { if (strcmp(setup->udfName, funcName) == 0) {
udf->resident = true; udf->resident = true;
break; break;
@ -248,11 +255,12 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
SUdfCallRequest *call = &request->call; SUdfCallRequest *call = &request->call;
fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64 , call->callType, call->udfHandle, request->seqNum); fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64, call->callType, call->udfHandle,
SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); request->seqNum);
SUdf * udf = handle->udf; SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
SUdf *udf = handle->udf;
SUdfResponse response = {0}; SUdfResponse response = {0};
SUdfResponse * rsp = &response; SUdfResponse *rsp = &response;
SUdfCallResponse *subRsp = &rsp->callRsp; SUdfCallResponse *subRsp = &rsp->callRsp;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -352,7 +360,7 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
SUdfTeardownRequest *teardown = &request->teardown; SUdfTeardownRequest *teardown = &request->teardown;
fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle);
SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
SUdf * udf = handle->udf; SUdf *udf = handle->udf;
bool unloadUdf = false; bool unloadUdf = false;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -409,15 +417,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) { if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) {
SConnectRsp connectRsp = {0}; SConnectRsp connectRsp = {0};
tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
int32_t now = taosGetTimestampSec(); int32_t now = taosGetTimestampSec();
int32_t delta = abs(now - connectRsp.svrTimestamp); int32_t delta = abs(now - connectRsp.svrTimestamp);
if (delta > 900) { if (delta > 900) {
msgInfo->code = TSDB_CODE_TIME_UNSYNCED; msgInfo->code = TSDB_CODE_TIME_UNSYNCED;
goto _return; goto _return;
} }
if (connectRsp.epSet.numOfEps == 0) { if (connectRsp.epSet.numOfEps == 0) {
msgInfo->code = TSDB_CODE_APP_ERROR; msgInfo->code = TSDB_CODE_APP_ERROR;
goto _return; goto _return;
@ -434,7 +441,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
goto _return; goto _return;
} }
SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
SUdf * udf = msgInfo->param; SUdf *udf = msgInfo->param;
udf->funcType = pFuncInfo->funcType; udf->funcType = pFuncInfo->funcType;
udf->scriptType = pFuncInfo->scriptType; udf->scriptType = pFuncInfo->scriptType;
udf->outputType = pFuncInfo->outputType; udf->outputType = pFuncInfo->outputType;
@ -487,7 +494,7 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
taosArrayPush(retrieveReq.pFuncNames, udfName); taosArrayPush(retrieveReq.pFuncNames, udfName);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void * pReq = rpcMallocCont(contLen); void *pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
taosArrayDestroy(retrieveReq.pFuncNames); taosArrayDestroy(retrieveReq.pFuncNames);
@ -522,7 +529,7 @@ int32_t udfdConnectToMnode() {
connReq.startTime = taosGetTimestampMs(); connReq.startTime = taosGetTimestampMs();
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq);
void * pReq = rpcMallocCont(contLen); void *pReq = rpcMallocCont(contLen);
tSerializeSConnectReq(pReq, contLen, &connReq); tSerializeSConnectReq(pReq, contLen, &connReq);
SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
@ -589,7 +596,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); strncpy(finishFuncName, processFuncName, sizeof(finishFuncName));
strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix));
uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc));
char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *mergeSuffix = "_merge"; char *mergeSuffix = "_merge";
strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName));
strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix));
@ -601,9 +608,10 @@ static bool udfdRpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER ||
code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING ||
code == TSDB_CODE_APP_IS_STOPPING) { code == TSDB_CODE_APP_IS_STOPPING) {
if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH ||
msgType == TDMT_SCH_MERGE_FETCH) {
return false; return false;
} }
return true; return true;
} else { } else {
return false; return false;
@ -663,7 +671,7 @@ int32_t udfdOpenClientRpc() {
rpcInit.parent = &global; rpcInit.parent = &global;
rpcInit.rfp = udfdRpcRfp; rpcInit.rfp = udfdRpcRfp;
rpcInit.compressSize = tsCompressMsgSize; rpcInit.compressSize = tsCompressMsgSize;
global.clientRpc = rpcOpen(&rpcInit); global.clientRpc = rpcOpen(&rpcInit);
if (global.clientRpc == NULL) { if (global.clientRpc == NULL) {
fnError("failed to init dnode rpc client"); fnError("failed to init dnode rpc client");
@ -684,6 +692,17 @@ void udfdOnWrite(uv_write_t *req, int status) {
if (status < 0) { if (status < 0) {
fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status)); fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status));
} }
// remove work from the connection work list
if (work->conn != NULL) {
SUvUdfWork **ppWork;
for (ppWork = &work->conn->pWorkList; *ppWork && (*ppWork != work); ppWork = &((*ppWork)->pWorkNext)) {
}
if (*ppWork == work) {
*ppWork = work->pWorkNext;
} else {
fnError("work not in conn any more");
}
}
taosMemoryFree(work->output.base); taosMemoryFree(work->output.base);
taosMemoryFree(work); taosMemoryFree(work);
taosMemoryFree(req); taosMemoryFree(req);
@ -692,10 +711,11 @@ void udfdOnWrite(uv_write_t *req, int status) {
void udfdSendResponse(uv_work_t *work, int status) { void udfdSendResponse(uv_work_t *work, int status) {
SUvUdfWork *udfWork = (SUvUdfWork *)(work->data); SUvUdfWork *udfWork = (SUvUdfWork *)(work->data);
uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); if (udfWork->conn != NULL) {
write_req->data = udfWork; uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t));
uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); write_req->data = udfWork;
uv_write(write_req, udfWork->conn->client, &udfWork->output, 1, udfdOnWrite);
}
taosMemoryFree(work); taosMemoryFree(work);
} }
@ -716,8 +736,8 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
buf->len = 0; buf->len = 0;
} }
} else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) { } else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) {
buf->base = ctx->inputBuf + ctx->inputLen; buf->base = ctx->inputBuf + ctx->inputLen;
buf->len = msgHeadSize - ctx->inputLen; buf->len = msgHeadSize - ctx->inputLen;
} else { } else {
ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap;
void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap);
@ -744,10 +764,15 @@ bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) {
} }
void udfdHandleRequest(SUdfdUvConn *conn) { void udfdHandleRequest(SUdfdUvConn *conn) {
uv_work_t * work = taosMemoryMalloc(sizeof(uv_work_t)); char *inputBuf = conn->inputBuf;
int32_t inputLen = conn->inputLen;
uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t));
SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork));
udfWork->client = conn->client; udfWork->conn = conn;
udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); udfWork->pWorkNext = conn->pWorkList;
conn->pWorkList = udfWork;
udfWork->input = uv_buf_init(inputBuf, inputLen);
conn->inputBuf = NULL; conn->inputBuf = NULL;
conn->inputLen = 0; conn->inputLen = 0;
conn->inputCap = 0; conn->inputCap = 0;
@ -758,13 +783,19 @@ void udfdHandleRequest(SUdfdUvConn *conn) {
void udfdPipeCloseCb(uv_handle_t *pipe) { void udfdPipeCloseCb(uv_handle_t *pipe) {
SUdfdUvConn *conn = pipe->data; SUdfdUvConn *conn = pipe->data;
SUvUdfWork* pWork = conn->pWorkList;
while (pWork != NULL) {
pWork->conn = NULL;
pWork = pWork->pWorkNext;
}
taosMemoryFree(conn->client); taosMemoryFree(conn->client);
taosMemoryFree(conn->inputBuf); taosMemoryFree(conn->inputBuf);
taosMemoryFree(conn); taosMemoryFree(conn);
} }
void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
fnDebug("udf read %zd bytes from client", nread); fnDebug("udfd read %zd bytes from client", nread);
if (nread == 0) return; if (nread == 0) return;
SUdfdUvConn *conn = client->data; SUdfdUvConn *conn = client->data;
@ -780,10 +811,10 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
} }
if (nread < 0) { if (nread < 0) {
fnError("Receive error %s", uv_err_name(nread));
if (nread == UV_EOF) { if (nread == UV_EOF) {
// TODO check more when close fnInfo("udfd pipe read EOF");
} else { } else {
fnError("Receive error %s", uv_err_name(nread));
} }
udfdUvHandleError(conn); udfdUvHandleError(conn);
} }
@ -799,6 +830,7 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
uv_pipe_init(global.loop, client, 0); uv_pipe_init(global.loop, client, 0);
if (uv_accept(server, (uv_stream_t *)client) == 0) { if (uv_accept(server, (uv_stream_t *)client) == 0) {
SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn));
ctx->pWorkList = NULL;
ctx->client = (uv_stream_t *)client; ctx->client = (uv_stream_t *)client;
ctx->inputBuf = 0; ctx->inputBuf = 0;
ctx->inputLen = 0; ctx->inputLen = 0;
@ -891,7 +923,7 @@ static int32_t udfdUvInit() {
} }
global.loop = loop; global.loop = loop;
if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit
uv_pipe_init(global.loop, &global.ctrlPipe, 1); uv_pipe_init(global.loop, &global.ctrlPipe, 1);
uv_pipe_open(&global.ctrlPipe, 0); uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
@ -966,10 +998,10 @@ int32_t udfdInitResidentFuncs() {
} }
global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN); global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN);
char* pSave = tsUdfdResFuncs; char *pSave = tsUdfdResFuncs;
char* token; char *token;
while ((token = strtok_r(pSave, ",", &pSave)) != NULL) { while ((token = strtok_r(pSave, ",", &pSave)) != NULL) {
char func[TSDB_FUNC_NAME_LEN+1] = {0}; char func[TSDB_FUNC_NAME_LEN + 1] = {0};
strncpy(func, token, TSDB_FUNC_NAME_LEN); strncpy(func, token, TSDB_FUNC_NAME_LEN);
fnInfo("udfd add resident function %s", func); fnInfo("udfd add resident function %s", func);
taosArrayPush(global.residentFuncs, func); taosArrayPush(global.residentFuncs, func);
@ -980,10 +1012,10 @@ int32_t udfdInitResidentFuncs() {
int32_t udfdDeinitResidentFuncs() { int32_t udfdDeinitResidentFuncs() {
for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) {
char* funcName = taosArrayGet(global.residentFuncs, i); char *funcName = taosArrayGet(global.residentFuncs, i);
SUdf** udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); SUdf **udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName));
if (udfInHash) { if (udfInHash) {
SUdf* udf = *udfInHash; SUdf *udf = *udfInHash;
if (udf->destroyFunc) { if (udf->destroyFunc) {
(udf->destroyFunc)(); (udf->destroyFunc)();
} }

6
source/libs/parser/inc/sql.y
View File

@ -194,7 +194,7 @@ db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C).
db_options(A) ::= db_options(B) TSDB_PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TSDB_PAGESIZE, &C); } db_options(A) ::= db_options(B) TSDB_PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TSDB_PAGESIZE, &C); }
db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); }
db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); }
db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } //db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); }
db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); }
db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); }
db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, C); } db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, C); }
@ -1070,5 +1070,5 @@ null_ordering_opt(A) ::= NULLS FIRST.
null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; }
%fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL %fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL
FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT STRING FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT
TIMES UPDATE VALUES VARIABLE VIEW WAL. STRICT STRING TIMES UPDATE VALUES VARIABLE VIEW WAL.

9
source/libs/parser/src/parInsertSql.c
View File

@ -1411,6 +1411,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
(*pNumOfRows) = 0; (*pNumOfRows) = 0;
char* pLine = NULL; char* pLine = NULL;
int64_t readLen = 0; int64_t readLen = 0;
bool firstLine = (pStmt->fileProcessing == false);
pStmt->fileProcessing = false; pStmt->fileProcessing = false;
while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) { while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) {
if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) {
@ -1418,6 +1419,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
} }
if (readLen == 0) { if (readLen == 0) {
firstLine = false;
continue; continue;
} }
@ -1431,6 +1433,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
strtolower(pLine, pLine); strtolower(pLine, pLine);
const char* pRow = pLine; const char* pRow = pLine;
code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token);
if (code && firstLine) {
firstLine = false;
code = 0;
continue;
}
} }
if (TSDB_CODE_SUCCESS == code && gotRow) { if (TSDB_CODE_SUCCESS == code && gotRow) {
@ -1442,6 +1449,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt,
pStmt->fileProcessing = true; pStmt->fileProcessing = true;
break; break;
} }
firstLine = false;
} }
taosMemoryFree(pLine); taosMemoryFree(pLine);

51
source/libs/parser/src/parTranslater.c
View File

@ -2837,7 +2837,7 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjectionList) { static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjectionList, SHashObj** pOutput) {
SHashObj* pUserAliasSet = taosHashInit(LIST_LENGTH(pProjectionList), SHashObj* pUserAliasSet = taosHashInit(LIST_LENGTH(pProjectionList),
taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
SNode* pProject = NULL; SNode* pProject = NULL;
@ -2849,13 +2849,17 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection
} }
taosHashPut(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias), &pExpr, POINTER_BYTES); taosHashPut(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias), &pExpr, POINTER_BYTES);
} }
taosHashCleanup(pUserAliasSet); if (NULL == pOutput) {
taosHashCleanup(pUserAliasSet);
} else {
*pOutput = pUserAliasSet;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) {
if (pSelect->isSubquery) { if (pSelect->isSubquery) {
return checkProjectAlias(pCxt, pSelect->pProjectionList); return checkProjectAlias(pCxt, pSelect->pProjectionList, NULL);
} }
return rewriteProjectAlias(pSelect->pProjectionList); return rewriteProjectAlias(pSelect->pProjectionList);
} }
@ -3899,8 +3903,7 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt
pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]",
pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, tsdbMaxKeep);
tsdbMaxKeep);
} }
if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) { if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) {
@ -4055,7 +4058,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbPrecisionOption(pCxt, pOptions); code = checkDbPrecisionOption(pCxt, pOptions);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbKeepOption(pCxt, pOptions); // use precision code = checkDbKeepOption(pCxt, pOptions); // use precision
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE);
@ -5515,9 +5518,24 @@ static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) {
tNameGetFullDbName(&name, pDbFName); tNameGetFullDbName(&name, pDbFName);
} }
static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* aliasName, int32_t len) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt; if (NULL == taosHashGet(pUserAliasSet, "_wstart", strlen("_wstart"))) {
SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); snprintf(aliasName, len, "%s", "_wstart");
return;
}
if (NULL == taosHashGet(pUserAliasSet, "ts", strlen("ts"))) {
snprintf(aliasName, len, "%s", "ts");
return;
}
do {
taosRandStr(aliasName, len - 1);
aliasName[len - 1] = '\0';
} while (NULL != taosHashGet(pUserAliasSet, aliasName, strlen(aliasName)));
return;
}
static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj* pUserAliasSet) {
SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0);
if (NULL == pSelect->pWindow || if (NULL == pSelect->pWindow ||
(QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) { (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -5527,7 +5545,7 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
strcpy(pFunc->functionName, "_wstart"); strcpy(pFunc->functionName, "_wstart");
strcpy(pFunc->node.aliasName, pFunc->functionName); getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName));
int32_t code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); int32_t code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)pFunc); nodesDestroyNode((SNode*)pFunc);
@ -5535,6 +5553,17 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) {
return code; return code;
} }
static int32_t addWstartTsToCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt;
SHashObj* pUserAliasSet = NULL;
int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet);
if (TSDB_CODE_SUCCESS == code) {
code = addWstartTsToCreateStreamQueryImpl(pSelect, pUserAliasSet);
}
taosHashCleanup(pUserAliasSet);
return code;
}
static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) { static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) {
if (NULL == pStmt->pTags) { if (NULL == pStmt->pTags) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -5637,7 +5666,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) {
static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) {
pCxt->createStream = true; pCxt->createStream = true;
int32_t code = addWstartTsToCreateStreamQuery(pStmt->pQuery); int32_t code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt);
} }

5753
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

6
source/libs/parser/test/parInitialCTest.cpp
View File

@ -48,7 +48,7 @@ TEST_F(ParserInitialCTest, createAccount) {
* | PRECISION {'ms' | 'us' | 'ns'} * | PRECISION {'ms' | 'us' | 'ns'}
* | REPLICA value * | REPLICA value
* | RETENTIONS ingestion_duration:keep_duration ... * | RETENTIONS ingestion_duration:keep_duration ...
* | STRICT {'off' | 'on'} * | STRICT {'off' | 'on'} // not support
* | WAL_LEVEL value * | WAL_LEVEL value
* | VGROUPS value * | VGROUPS value
* | SINGLE_STABLE {0 | 1} * | SINGLE_STABLE {0 | 1}
@ -216,7 +216,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY);
addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY);
addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY);
setDbStrictaFunc(1); // setDbStrictaFunc(1);
setDbWalLevelFunc(2); setDbWalLevelFunc(2);
setDbVgroupsFunc(100); setDbVgroupsFunc(100);
setDbSingleStableFunc(1); setDbSingleStableFunc(1);
@ -244,7 +244,7 @@ TEST_F(ParserInitialCTest, createDatabase) {
"PRECISION 'ns' " "PRECISION 'ns' "
"REPLICA 3 " "REPLICA 3 "
"RETENTIONS 15s:7d,1m:21d,15m:500d " "RETENTIONS 15s:7d,1m:21d,15m:500d "
"STRICT 'on' " // "STRICT 'on' "
"WAL_LEVEL 2 " "WAL_LEVEL 2 "
"VGROUPS 100 " "VGROUPS 100 "
"SINGLE_STABLE 1 " "SINGLE_STABLE 1 "

2
source/libs/qworker/inc/qwInt.h
View File

@ -31,7 +31,7 @@ extern "C" {
#define QW_DEFAULT_SCHEDULER_NUMBER 100 #define QW_DEFAULT_SCHEDULER_NUMBER 100
#define QW_DEFAULT_TASK_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000
#define QW_DEFAULT_SCH_TASK_NUMBER 3000 #define QW_DEFAULT_SCH_TASK_NUMBER 500
#define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_SHORT_RUN_TIMES 2
#define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_DEFAULT_HEARTBEAT_MSEC 5000
#define QW_SCH_TIMEOUT_MSEC 180000 #define QW_SCH_TIMEOUT_MSEC 180000

2
source/libs/qworker/src/qwUtil.c
View File

@ -74,6 +74,8 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) {
SQWSchStatus newSch = {0}; SQWSchStatus newSch = {0};
newSch.tasksHash = newSch.tasksHash =
taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
newSch.hbBrokenTs = taosGetTimestampMs();
if (NULL == newSch.tasksHash) { if (NULL == newSch.tasksHash) {
QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum);
QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);

2
source/libs/scheduler/src/schStatus.c
View File

@ -64,7 +64,7 @@ _return:
int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) {
SSchJob* pJob = schAcquireJob(jobId); SSchJob* pJob = schAcquireJob(jobId);
if (NULL == pJob) { if (NULL == pJob) {
qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId);
SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR);
} }

2
source/libs/scheduler/src/scheduler.c
View File

@ -157,7 +157,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) {
SSchJob *pJob = schAcquireJob(*jobId); SSchJob *pJob = schAcquireJob(*jobId);
if (NULL == pJob) { if (NULL == pJob) {
qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId);
return; return;
} }

2
source/libs/sync/inc/syncInt.h
View File

@ -215,7 +215,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode);
int32_t syncNodeStartStandBy(SSyncNode* pSyncNode); int32_t syncNodeStartStandBy(SSyncNode* pSyncNode);
void syncNodeClose(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode);
void syncNodePreClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode);
int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak); int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t *seq);
int32_t syncNodeRestore(SSyncNode* pSyncNode); int32_t syncNodeRestore(SSyncNode* pSyncNode);
void syncHbTimerDataFree(SSyncHbTimerData* pData); void syncHbTimerDataFree(SSyncHbTimerData* pData);

31
source/libs/sync/src/syncMain.c
View File

@ -151,7 +151,7 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) {
} }
syncNodeStartHeartbeatTimer(pSyncNode); syncNodeStartHeartbeatTimer(pSyncNode);
//syncNodeReplicate(pSyncNode); // syncNodeReplicate(pSyncNode);
} }
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
@ -218,6 +218,26 @@ int32_t syncLeaderTransfer(int64_t rid) {
return ret; return ret;
} }
int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) {
SSyncNode* pNode = syncNodeAcquire(rid);
if (pNode == NULL) return -1;
SRpcMsg rpcMsg = {0};
int32_t ret = syncRespMgrGetAndDel(pNode->pSyncRespMgr, seq, &rpcMsg.info);
rpcMsg.code = TSDB_CODE_SYN_TIMEOUT;
syncNodeRelease(pNode);
if (ret == 1) {
sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle,
rpcMsg.info.ahandle);
rpcSendResponse(&rpcMsg);
return 0;
} else {
sError("no message handle to send timeout response, seq:%" PRId64, seq);
return -1;
}
}
SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) { SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) {
SyncIndex minMatchIndex = SYNC_INDEX_INVALID; SyncIndex minMatchIndex = SYNC_INDEX_INVALID;
@ -538,7 +558,7 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) {
pMsg->newLeaderId.vgId = pSyncNode->vgId; pMsg->newLeaderId.vgId = pSyncNode->vgId;
pMsg->newNodeInfo = newLeader; pMsg->newNodeInfo = newLeader;
int32_t ret = syncNodePropose(pSyncNode, &rpcMsg, false); int32_t ret = syncNodePropose(pSyncNode, &rpcMsg, false, NULL);
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);
return ret; return ret;
} }
@ -670,19 +690,19 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
} }
int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq) {
SSyncNode* pSyncNode = syncNodeAcquire(rid); SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
sError("sync propose error"); sError("sync propose error");
return -1; return -1;
} }
int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak, seq);
syncNodeRelease(pSyncNode); syncNodeRelease(pSyncNode);
return ret; return ret;
} }
int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) {
if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) {
terrno = TSDB_CODE_SYN_NOT_LEADER; terrno = TSDB_CODE_SYN_NOT_LEADER;
sNError(pSyncNode, "sync propose not leader, %s, type:%s", syncStr(pSyncNode->state), TMSG_INFO(pMsg->msgType)); sNError(pSyncNode, "sync propose not leader, %s, type:%s", syncStr(pSyncNode->state), TMSG_INFO(pMsg->msgType));
@ -739,6 +759,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) {
(void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum);
} }
if (seq != NULL) *seq = seqNum;
return code; return code;
} }
} }

9
source/libs/sync/src/syncPipeline.c
View File

@ -594,6 +594,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
int count = 0; int count = 0;
int64_t firstIndex = -1; int64_t firstIndex = -1;
SyncTerm term = -1; SyncTerm term = -1;
int64_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff));
for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) {
int64_t pos = index % pMgr->size; int64_t pos = index % pMgr->size;
@ -620,7 +621,10 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
retried = true; retried = true;
if (firstIndex == -1) firstIndex = index; if (firstIndex == -1) firstIndex = index;
count++;
if (batchSize <= count++) {
break;
}
} }
ret = 0; ret = 0;
@ -800,8 +804,9 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode
int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) {
ASSERT(pMgr->restored); ASSERT(pMgr->restored);
SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; SRaftId* pDestId = &pNode->replicasId[pMgr->peerId];
int32_t batchSize = TMAX(1, pMgr->size / 20); int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff));
int32_t count = 0; int32_t count = 0;
int64_t nowMs = taosGetMonoTimestampMs(); int64_t nowMs = taosGetMonoTimestampMs();
int64_t limit = pMgr->size >> 1; int64_t limit = pMgr->size >> 1;

30
source/libs/sync/src/syncRespMgr.c
View File

@ -35,11 +35,16 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
pObj->seqNum = 0; pObj->seqNum = 0;
taosThreadMutexInit(&(pObj->mutex), NULL); taosThreadMutexInit(&(pObj->mutex), NULL);
SSyncNode *pNode = pObj->data;
sTrace("vgId:%d, create resp manager", pNode->vgId);
return pObj; return pObj;
} }
void syncRespMgrDestroy(SSyncRespMgr *pObj) { void syncRespMgrDestroy(SSyncRespMgr *pObj) {
if (pObj != NULL) { if (pObj != NULL) {
SSyncNode *pNode = pObj->data;
sTrace("vgId:%d, destroy resp manager", pNode->vgId);
taosThreadMutexLock(&pObj->mutex); taosThreadMutexLock(&pObj->mutex);
taosHashCleanup(pObj->pRespHash); taosHashCleanup(pObj->pRespHash);
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
@ -81,6 +86,8 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub) {
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
return 1; // get one object return 1; // get one object
} else {
sNError(pObj->data, "get message handle, no object of seq:%" PRIu64, seq);
} }
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
@ -99,6 +106,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *p
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
return 1; // get one object return 1; // get one object
} else {
sNError(pObj->data, "get-and-del message handle, no object of seq:%" PRIu64, seq);
} }
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
@ -114,7 +123,7 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) {
SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t)); SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t));
if (delIndexArray == NULL) return; if (delIndexArray == NULL) return;
sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); sDebug("vgId:%d, resp manager begin clean by ttl", pSyncNode->vgId);
while (pStub) { while (pStub) {
size_t len; size_t len;
void *key = taosHashGetKey(pStub, &len); void *key = taosHashGetKey(pStub, &len);
@ -143,34 +152,39 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) {
// TODO: and make rpcMsg body, call commit cb // TODO: and make rpcMsg body, call commit cb
// pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta); // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta);
SRpcMsg rpcMsg = {.info = pStub->rpcMsg.info, .code = TSDB_CODE_SYN_TIMEOUT};
pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; sInfo("vgId:%d, message handle:%p expired, type:%s ahandle:%p", pSyncNode->vgId, rpcMsg.info.handle,
if (pStub->rpcMsg.info.handle != NULL) { TMSG_INFO(pStub->rpcMsg.msgType), rpcMsg.info.ahandle);
tmsgSendRsp(&pStub->rpcMsg); rpcSendResponse(&rpcMsg);
}
} }
pStub = taosHashIterate(pObj->pRespHash, pStub); pStub = taosHashIterate(pObj->pRespHash, pStub);
} }
int32_t arraySize = taosArrayGetSize(delIndexArray); int32_t arraySize = taosArrayGetSize(delIndexArray);
sDebug("vgId:%d, resp mgr end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize); sDebug("vgId:%d, resp manager end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize);
for (int32_t i = 0; i < arraySize; ++i) { for (int32_t i = 0; i < arraySize; ++i) {
uint64_t *pSeqNum = taosArrayGet(delIndexArray, i); uint64_t *pSeqNum = taosArrayGet(delIndexArray, i);
taosHashRemove(pObj->pRespHash, pSeqNum, sizeof(uint64_t)); taosHashRemove(pObj->pRespHash, pSeqNum, sizeof(uint64_t));
sDebug("vgId:%d, resp mgr clean by ttl, seq:%" PRId64 "", pSyncNode->vgId, *pSeqNum); sDebug("vgId:%d, resp manager clean by ttl, seq:%" PRId64, pSyncNode->vgId, *pSeqNum);
} }
taosArrayDestroy(delIndexArray); taosArrayDestroy(delIndexArray);
} }
void syncRespCleanRsp(SSyncRespMgr *pObj) { void syncRespCleanRsp(SSyncRespMgr *pObj) {
SSyncNode *pNode = pObj->data;
sTrace("vgId:%d, clean all rsp", pNode->vgId);
taosThreadMutexLock(&pObj->mutex); taosThreadMutexLock(&pObj->mutex);
syncRespCleanByTTL(pObj, -1, true); syncRespCleanByTTL(pObj, -1, true);
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);
} }
void syncRespClean(SSyncRespMgr *pObj) { void syncRespClean(SSyncRespMgr *pObj) {
SSyncNode *pNode = pObj->data;
sTrace("vgId:%d, clean rsp by ttl", pNode->vgId);
taosThreadMutexLock(&pObj->mutex); taosThreadMutexLock(&pObj->mutex);
syncRespCleanByTTL(pObj, pObj->ttl, false); syncRespCleanByTTL(pObj, pObj->ttl, false);
taosThreadMutexUnlock(&pObj->mutex); taosThreadMutexUnlock(&pObj->mutex);

2
source/libs/sync/test/syncConfigChangeSnapshotTest.cpp
View File

@ -337,7 +337,7 @@ int main(int argc, char** argv) {
if (alreadySend < writeRecordNum) { if (alreadySend < writeRecordNum) {
SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
int32_t ret = syncPropose(rid, pRpcMsg, false); int32_t ret = syncPropose(rid, pRpcMsg, false, NULL);
if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
sTrace("%s value%d write not leader", s, alreadySend); sTrace("%s value%d write not leader", s, alreadySend);
} else { } else {

2
source/libs/sync/test/syncConfigChangeTest.cpp
View File

@ -249,7 +249,7 @@ int main(int argc, char** argv) {
if (alreadySend < writeRecordNum) { if (alreadySend < writeRecordNum) {
SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
int32_t ret = syncPropose(rid, pRpcMsg, false); int32_t ret = syncPropose(rid, pRpcMsg, false, NULL);
if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
sTrace("%s value%d write not leader", s, alreadySend); sTrace("%s value%d write not leader", s, alreadySend);
} else { } else {

2
source/libs/sync/test/syncReplicateTest.cpp
View File

@ -189,7 +189,7 @@ int main(int argc, char** argv) {
if (alreadySend < writeRecordNum) { if (alreadySend < writeRecordNum) {
SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
int32_t ret = syncPropose(rid, pRpcMsg, false); int32_t ret = syncPropose(rid, pRpcMsg, false, NULL);
if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
sTrace("%s value%d write not leader", s, alreadySend); sTrace("%s value%d write not leader", s, alreadySend);
} else { } else {

2
source/libs/sync/test/syncTestTool.cpp
View File

@ -396,7 +396,7 @@ int main(int argc, char** argv) {
if (alreadySend < writeRecordNum) { if (alreadySend < writeRecordNum) {
SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex);
int32_t ret = syncPropose(rid, pRpcMsg, false); int32_t ret = syncPropose(rid, pRpcMsg, false, NULL);
if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) {
sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait);
} else { } else {

6
source/libs/transport/inc/transportInt.h
View File

@ -47,10 +47,8 @@ typedef struct {
char label[TSDB_LABEL_LEN]; char label[TSDB_LABEL_LEN];
char user[TSDB_UNI_LEN]; // meter ID char user[TSDB_UNI_LEN]; // meter ID
int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size
int8_t encryption; // encrypt or not int8_t encryption; // encrypt or not
int32_t retryLimit; // retry limit
int32_t retryInterval; // retry interval ms
int32_t retryMinInterval; // retry init interval int32_t retryMinInterval; // retry init interval
int32_t retryStepFactor; // retry interval factor int32_t retryStepFactor; // retry interval factor

6
source/libs/transport/src/trans.c
View File

@ -47,9 +47,11 @@ void* rpcOpen(const SRpcInit* pInit) {
} }
pRpc->compressSize = pInit->compressSize; pRpc->compressSize = pInit->compressSize;
if (pRpc->compressSize < 0) {
pRpc->compressSize = -1;
}
pRpc->encryption = pInit->encryption; pRpc->encryption = pInit->encryption;
pRpc->retryLimit = pInit->retryLimit;
pRpc->retryInterval = pInit->retryInterval;
pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval
pRpc->retryStepFactor = pInit->retryStepFactor; pRpc->retryStepFactor = pInit->retryStepFactor;

7
source/libs/transport/src/transCli.c
View File

@ -604,6 +604,7 @@ static int32_t allocConnRef(SCliConn* conn, bool update) {
static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) {
if (update) { if (update) {
transReleaseExHandle(transGetRefMgt(), conn->refId);
transRemoveExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId);
conn->refId = -1; conn->refId = -1;
} }
@ -956,7 +957,6 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) {
SCliConn* conn = exh->handle; SCliConn* conn = exh->handle;
transReleaseExHandle(transGetRefMgt(), refId); transReleaseExHandle(transGetRefMgt(), refId);
tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn);
if (T_REF_VAL_GET(conn) == 2) { if (T_REF_VAL_GET(conn) == 2) {
@ -1574,6 +1574,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
pCtx->retryStep = 0; pCtx->retryStep = 0;
pCtx->retryInit = true; pCtx->retryInit = true;
pCtx->retryCode = TSDB_CODE_SUCCESS; pCtx->retryCode = TSDB_CODE_SUCCESS;
// already retry, not use handle specified by app;
pMsg->msg.info.handle = 0;
} }
if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) {
@ -1595,7 +1598,7 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
} else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR ||
code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED ||
code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING ||
code == TSDB_CODE_APP_IS_STOPPING) { code == TSDB_CODE_APP_IS_STOPPING || code == TSDB_CODE_VND_STOPPED) {
tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen);
noDelay = cliResetEpset(pCtx, pResp, true); noDelay = cliResetEpset(pCtx, pResp, true);
transFreeMsg(pResp->pCont); transFreeMsg(pResp->pCont);

2
source/os/src/osEnv.c
View File

@ -37,7 +37,7 @@ float tsNumOfCores = 0;
int64_t tsTotalMemoryKB = 0; int64_t tsTotalMemoryKB = 0;
char *tsProcPath = NULL; char *tsProcPath = NULL;
char tsSIMDEnable = 0; char tsSIMDBuiltins = 0;
char tsSSE42Enable = 0; char tsSSE42Enable = 0;
char tsAVXEnable = 0; char tsAVXEnable = 0;
char tsAVX2Enable = 0; char tsAVX2Enable = 0;

4
source/os/src/osFile.c
View File

@ -464,7 +464,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) {
#if FILE_WITH_LOCK #if FILE_WITH_LOCK
taosThreadRwlockWrlock(&(pFile->rwlock)); taosThreadRwlockWrlock(&(pFile->rwlock));
#endif #endif
assert(pFile->fd >= 0); // Please check if you have closed the file. if (pFile->fd < 0) {
return 0;
}
int64_t nleft = count; int64_t nleft = count;
int64_t nwritten = 0; int64_t nwritten = 0;

10
source/os/src/osSysinfo.c
View File

@ -489,11 +489,11 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) {
#ifdef _TD_X86_ #ifdef _TD_X86_
// Since the compiler is not support avx/avx2 instructions, the global variables always need to be // Since the compiler is not support avx/avx2 instructions, the global variables always need to be
// set to be false // set to be false
#if __AVX__ || __AVX2__ //#if __AVX__ || __AVX2__
tsSIMDEnable = true; // tsSIMDBuiltins = true;
#else //#else
tsSIMDEnable = false; // tsSIMDBuiltins = false;
#endif //#endif
uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;

1
source/util/src/terror.c
View File

@ -281,6 +281,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists
TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB, "Stream temporarily does not support source db having replica > 1")
// mnode-sma // mnode-sma
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists")

Some files were not shown because too many files have changed in this diff Show More