diff --git a/cmake/cmake.version b/cmake/cmake.version index 5bb1c61ac2..156e99bd03 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -11,6 +11,10 @@ ELSE () SET(TD_VER_COMPATIBLE "3.0.0.0") ENDIF () +IF (TD_PRODUCT_NAME) + ADD_DEFINITIONS(-DTD_PRODUCT_NAME="${TD_PRODUCT_NAME}") +ENDIF () + find_program(HAVE_GIT NAMES git) IF (DEFINED GITINFO) diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index 37077ef63b..072e9f857d 100644 Binary files a/deps/arm/dm_static/libdmodule.a and b/deps/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/arm/dm_static/libdmodule.a b/deps/darwin/arm/dm_static/libdmodule.a index 246b2247af..2bede61caa 100644 Binary files a/deps/darwin/arm/dm_static/libdmodule.a and b/deps/darwin/arm/dm_static/libdmodule.a differ diff --git a/deps/darwin/x64/dm_static/libdmodule.a b/deps/darwin/x64/dm_static/libdmodule.a index 8745f57636..ae1abc5dfc 100644 Binary files a/deps/darwin/x64/dm_static/libdmodule.a and b/deps/darwin/x64/dm_static/libdmodule.a differ diff --git a/deps/mips/dm_static/libdmodule.a b/deps/mips/dm_static/libdmodule.a index 855a6a41d9..868ac62d3e 100644 Binary files a/deps/mips/dm_static/libdmodule.a and b/deps/mips/dm_static/libdmodule.a differ diff --git a/deps/x86/dm_static/libdmodule.a b/deps/x86/dm_static/libdmodule.a index 6a3c0d45c2..658bb95c27 100644 Binary files a/deps/x86/dm_static/libdmodule.a and b/deps/x86/dm_static/libdmodule.a differ diff --git a/docs/en/08-client-libraries/03-cpp.mdx b/docs/en/08-client-libraries/03-cpp.mdx index dda436f553..97fea49902 100644 --- a/docs/en/08-client-libraries/03-cpp.mdx +++ b/docs/en/08-client-libraries/03-cpp.mdx @@ -451,6 +451,101 @@ In addition to writing data using the SQL method or the parameter binding API, w - Within _reqid interfaces can track the entire call chain by passing the reqid parameter. ### Subscription API +- `const char *tmq_err2str(int32_t code)` + + **Description** + - This interface is used to convert error codes for data subscriptions into error messages + + **Parameter description** + - code: error code + + **Return value** + - non NULL, return error message, error message may be empty + + +- `tmq_conf_t *tmq_conf_new()` +- `tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value)` +- `void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param)` +- `void tmq_conf_destroy(tmq_conf_t *conf)` + + tmq_conf_res_t defined as follows: + ``` + typedef enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, + } tmq_conf_res_t; + ``` + + commit callback function defined as follows: + ``` + typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param)) + ``` + **Description** + - tmq_conf_new : create a tmq_conf_t structure to configure consumption parameters + - tmq_conf_set : set configuration, configuration is key-value pair + - tmq_conf_set_auto_commit_cb : set auto commit callback function + - tmq_conf_destroy : destroy tmq_conf_t structure + + **Parameter description** + - tmq_conf_set : key is parameter name，value is parameter value + - tmq_conf_set_auto_commit_cb : cb is callback function, param is callback function parameter + + **Return value** + - tmq_conf_new: structure of tmq_conf_t, NULL failed + - tmq_conf_set: tmq_conf_res_t, TMQ_CONF_OK means success, others means failure + + +- `tmq_list_t *tmq_list_new()` +- `int32_t tmq_list_append(tmq_list_t *, const char *)` +- `void tmq_list_destroy(tmq_list_t *)` +- `int32_t tmq_list_get_size(const tmq_list_t *)` +- `char **tmq_list_to_c_array(const tmq_list_t *)` + + **Description** + - tmq_list_new : build a tmq_list_t constructure, used to save topic + - tmq_list_append : add topic to tmq_list_t + - tmq_list_destroy : destroy tmq_list_t + - tmq_list_get_size : get size of tmq_list_t + - tmq_list_to_c_array : convert tmq_list_t to c array, element is string pointer + + **Return value** + - tmq_list_new : structure of tmq_list_t, tmq_list_t is a list of strings, NULL failed + - tmq_list_append : zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_list_get_size : size of tmq_list_t, -1 failed + - tmq_list_to_c_array : c array, element is pointer of string, NULL failed + +- `tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen)` +- `int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list)` +- `int32_t tmq_unsubscribe(tmq_t *tmq)` +- `int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topic_list)` +- `TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout)` +- `int32_t tmq_consumer_close(tmq_t *tmq)` + + **Description** + - tmq_consumer_new : build a tmq_t constructure, need to be used with tmq_consumer_close + - tmq_subscribe : subscribe topic, need to be used with tmq_unsubscribe + - tmq_unsubscribe : unsubscribe topic, need to be used with tmq_subscribe + - tmq_subscription : obtain a list of topics subscribed by consumer + - tmq_consumer_poll : used to consume data + - tmq_consumer_close : clost tmq_t, need to be used with tmq_consumer_new + + **Parameter description** + - conf: sed to configure consume parameters + - errstr: The error information is stored in this string. Allocation and release of memory are the responsibility of the caller + - errstenLen: the length of errstr + - tmq: structure of tmq_t returned by tmq_consumer_new + - topic_list: a list of topics subscribed by consumers，need to be freed by tmq_list_destroy + - timeout: the timeout time, measured in milliseconds, indicates how long it takes for data to expire. If it is negative, it will default to 1 second + + **Return value** + - tmq_consumer_new: structure of tmq_t, NULL failed + - tmq_subscribe: zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_unsubscribe: zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_subscription: zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_consumer_poll: structure of TAOS_RES(same like taos_query), NULL if there is no data + - tmq_consumer_close: zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - `int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment)` - `void tmq_free_assignment(tmq_topic_assignment* pAssignment)` @@ -474,6 +569,7 @@ In addition to writing data using the SQL method or the parameter binding API, w **Return value** - zero success,none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)` **Function description** @@ -482,6 +578,7 @@ In addition to writing data using the SQL method or the parameter binding API, w **Return value** - the value of committed offset, -2147467247 means no committed value, Other values less than 0 indicate failure + - `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)` - `void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param)` - `int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` @@ -499,6 +596,7 @@ In addition to writing data using the SQL method or the parameter binding API, w **Return value** - zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - `int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId)` **Function description** @@ -507,6 +605,7 @@ In addition to writing data using the SQL method or the parameter binding API, w **Return value** - the current consumption location, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - `int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` **Function description** @@ -514,25 +613,52 @@ In addition to writing data using the SQL method or the parameter binding API, w **Return value** - zero success, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + + +- `int64_t tmq_get_vgroup_offset(TAOS_RES* res)` +- `int32_t tmq_get_vgroup_id(TAOS_RES *res)` + + **Description** + - tmq_get_vgroup_offset : Obtain the starting offset of the consumed data + - tmq_get_vgroup_id : Obtain the vgroup id of the consumed data -- `int32_t int64_t tmq_get_vgroup_offset(TAOS_RES* res)` - - **Function description** - - Obtain the starting offset of the consumed data - **Parameter description** - - msg：Message consumed - - **Return value** - - the starting offset of the consumed data, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - msg : Message consumed -- `int32_t int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics)` - - **Function description** - - Obtain a list of topics subscribed by consumers - - **Parameter description** - - topics: a list of topics subscribed by consumers，need to be freed by tmq_list_destroy - **Return value** - - zero success，none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_get_vgroup_offset : the starting offset of the consumed data, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + - tmq_get_vgroup_id : vgroup id of result, none zero failed, wrong message can be obtained through `char *tmq_err2str(int32_t code)` + + +- `TAOS *tmq_get_connect(tmq_t *tmq)` +- `const char *tmq_get_table_name(TAOS_RES *res)` +- `tmq_res_t tmq_get_res_type(TAOS_RES *res)` +- `const char *tmq_get_topic_name(TAOS_RES *res)` +- `const char *tmq_get_db_name(TAOS_RES *res)` + + tmq_res_t the type of consumed result, defined as follows: + ``` + typedef enum tmq_res_t { + TMQ_RES_INVALID = -1, // invalid + TMQ_RES_DATA = 1, // data + TMQ_RES_TABLE_META = 2, // meta + TMQ_RES_METADATA = 3 // data & meta + } tmq_res_t; + ``` + **Description** + - tmq_get_connect : when creating a consumer, a link will be automatically established and saved in the tmq_t structure. This interface allows users to obtain link information(same like taos_connect) from the tmq_t structure + - tmq_get_table_name : get the table name of result + - tmq_get_res_type : get the type of result + - tmq_get_topic_name : get the topic name of result + - tmq_get_db_name : get the db name of result + + **Parameter description** + - tmq : tmq_t structure created by tmq_consumer_new + - res : TAOS_RES structure returned by tmq_consumer_poll + + **Return value** + - tmq_get_connect : connection info in tmq, NULL if failed + - tmq_get_table_name : table name of result, NULL if failed + - tmq_get_res_type : result type tmq_res_t + - tmq_get_topic_name : topic name of result, NULL if failed + - tmq_get_db_name : db name of result, NULL if failed \ No newline at end of file diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 3f93f69f0f..e7cefc1d7a 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -80,7 +80,7 @@ If a stream is created with PARTITION BY clause and SUBTABLE clause, the name of CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); ``` -IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name(Starting from 3.2.3.0, in order to avoid the expression in subtable being unable to distinguish between different subtables, add '_groupId' to the end of subtable name). +IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name(Starting from 3.2.3.0, in order to avoid the expression in subtable being unable to distinguish between different subtables, add '_stableName_groupId' to the end of subtable name). If the output length exceeds the limitation of TDengine(192), the name will be truncated. If the generated name is occupied by some other table, the creation and writing of the new subtable will be failed. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index fd6d3a2930..5e4eadcceb 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -421,7 +421,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | Maximum number of vnodes per dnode | | Value Range | 0-4096 | -| Default Value | 2x the CPU cores | +| Default Value | 2x the CPU cores plus 5 | ## Performance Tuning diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index d4487e6148..f295e57bb5 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.3.0.3 + + + ## 3.3.0.0 diff --git a/docs/examples/R/connect_native.r b/docs/examples/R/connect_native.r index bd764c3ec4..60d245781e 100644 --- a/docs/examples/R/connect_native.r +++ b/docs/examples/R/connect_native.r @@ -1,5 +1,5 @@ if (! "RJDBC" %in% installed.packages()[, "Package"]) { - install.packages('RJDBC', repos='http://cran.us.r-project.org') + install.packages('RJDBC', repos='http://mirrors.tuna.tsinghua.edu.cn/CRAN') } # ANCHOR: demo diff --git a/docs/zh/08-connector/10-cpp.mdx b/docs/zh/08-connector/10-cpp.mdx index e5dc2ec8c8..f03777ac4b 100644 --- a/docs/zh/08-connector/10-cpp.mdx +++ b/docs/zh/08-connector/10-cpp.mdx @@ -528,10 +528,103 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 - 带_reqid的接口可以通过传递reqid参数来追踪整个的调用链。 ### 数据订阅 API +- `const char *tmq_err2str(int32_t code)` + + **功能说明** + - 该接口用于将数据订阅的错误码转换为错误信息 + + **参数说明** + - code: 数据订阅的错误码 + + **返回值** + - 非NULL，返回错误信息，错误信息可能为空字符串 + + +- `tmq_conf_t *tmq_conf_new()` +- `tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value)` +- `void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb, void *param)` +- `void tmq_conf_destroy(tmq_conf_t *conf)` + tmq_conf_res_t 错误码定义如下： + ``` + typedef enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, + } tmq_conf_res_t; + ``` + 设置自动提交回调函数的定义如下： + ``` + typedef void(tmq_commit_cb(tmq_t *tmq, int32_t code, void *param)) + ``` + **功能说明** + - tmq_conf_new 接口用于创建一个 tmq_conf_t 结构体，用于配置消费参数。 + - tmq_conf_set 接口用于设置消费参数。 + - tmq_conf_set_auto_commit_cb 接口用于设置自动提交回调函数。 + - tmq_conf_destroy 接口用于销毁 tmq_conf_t 结构体。 + + **参数说明** + - tmq_conf_set : key 为参数名，value 为参数值 + - tmq_conf_set_auto_commit_cb : cb 回调函数, param 回调函数参数 + + **返回值** + - tmq_conf_new: 配置 tmq_conf_t 类型指针, NULL 失败 + - tmq_conf_set: 结果 tmq_conf_res_t 类型, TMQ_CONF_OK 成功, 其他失败 + +- `tmq_list_t *tmq_list_new()` +- `int32_t tmq_list_append(tmq_list_t *, const char *)` +- `void tmq_list_destroy(tmq_list_t *)` +- `int32_t tmq_list_get_size(const tmq_list_t *)` +- `char **tmq_list_to_c_array(const tmq_list_t *)` + + **功能说明** + - tmq_list_new 接口用于创建一个 tmq_list_t 结构体，用于存储订阅的 topic。 + - tmq_list_append 接口用于向 tmq_list_t 结构体中添加一个 topic。 + - tmq_list_destroy 接口用于销毁 tmq_list_t 结构体，tmq_list_new 的结果需要通过该接口销毁。 + - tmq_list_get_size 接口用于获取 tmq_list_t 结构体中 topic 的个数。 + - tmq_list_to_c_array 接口用于将 tmq_list_t 结构体转换为 C 数组，数组每个元素为字符串指针。 + + **返回值** + - tmq_list_new : 返回 tmq_list_t 结果指针, tmq_list_t 是一个数组，每个元素是一个字符串, NULL 失败 + - tmq_list_append : 返回 0 表示成功，非0表示失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - tmq_list_get_size : 返回 tmq_list_t 结构体中 topic 的个数, -1 失败 + - tmq_list_to_c_array : 返回 c 数组, 每个元素是字符串指针, NULL 失败 + + +- `tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errstrLen)` +- `int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list)` +- `int32_t tmq_unsubscribe(tmq_t *tmq)` +- `int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topic_list)` +- `TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout)` +- `int32_t tmq_consumer_close(tmq_t *tmq)` + + **功能说明** + - tmq_consumer_new 接口用于创建一个 tmq_t 结构体，用于消费数据，消费完数据后需调用 tmq_consumer_close 关闭消费者。 + - tmq_subscribe 接口用于订阅 topic 列表，消费完数据后，需调用 tmq_subscribe 取消订阅。 + - tmq_unsubscribe 接口用于取消订阅的 topic 列表。需与 tmq_subscribe 配合使用。 + - tmq_subscription 接口用于获取订阅的 topic 列表。 + - tmq_consumer_poll 接口用于轮询消费数据，每一个消费者，只能单线程调用该接口。 + - tmq_consumer_close 接口用于关闭 tmq_t 结构体。需与 tmq_consumer_new 配合使用。 + + **参数说明** + - conf: 参数用于配置消费参数 + - errstr: 错误信息存储在这个字符串中，需自定分配内存，释放内存由调用者负责 + - errstenLen: errstr 字符串的长度 + - tmq: tmq_consumer_new 函数返回的 tmq_t 结构体 + - topic_list: topic 列表 + - timeout: 超时时间，单位为毫秒，表示多久没数据的话自动返回 NULL，负数的话默认超时1秒 + + **返回值** + - tmq_consumer_new 返回 tmq_t 结构体，失败返回 NULL + - tmq_subscribe 返回错误码，0 表示成功，非0表示失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - tmq_unsubscribe 返回错误码，0 表示成功，非0表示失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - tmq_subscription 返回错误码，0 表示成功，非0表示失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - tmq_consumer_poll 返回 TAOS_RES 结构体，NULL 表示没有数据，非 NULL 表示有数据，TAOS_RES 结果和 taos_query 返回结果一致，可通过查询的各种接口获取 TAOS_RES 里的信息，比如 schema 等。 + - tmq_consumer_close 返回错误码，0 表示成功，非0表示失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - `int32_t tmq_get_topic_assignment(tmq_t *tmq, const char *pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment)` - `void tmq_free_assignment(tmq_topic_assignment* pAssignment)` - + tmq_topic_assignment结构体定义如下： ```c typedef struct tmq_topic_assignment { @@ -541,6 +634,7 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 int64_t end; } tmq_topic_assignment; ``` + **功能说明** - tmq_get_topic_assignment 接口返回当前consumer分配的vgroup的信息，每个vgroup的信息包括vgId，wal的最大最小offset，以及当前消费到的offset。 @@ -551,65 +645,97 @@ TDengine 的异步 API 均采用非阻塞调用模式。应用程序可以用多 **返回值** - 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息。 + - `int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId)` - + **功能说明** - 获取当前 consumer 在某个 topic 和 vgroup上的 commit 位置。 **返回值** - 当前commit的位置，-2147467247表示没有消费进度，其他小于0的值表示失败，错误码就是返回值 + - `int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg)` - `void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param)` - `int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` - `void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param)` - + **功能说明** - + - commit接口分为两种类型，每种类型有同步和异步接口： - - 第一种类型：根据消息提交，提交消息里的进度，如果消息传NULL，提交当前consumer所有消费的vgroup的当前进度 : tmq_commit_sync/tmq_commit_async - - 第二种类型：根据某个topic的某个vgroup的offset提交 : tmq_commit_offset_sync/tmq_commit_offset_async - + - 第一种类型：根据消息提交，提交消息里的进度，如果消息传NULL，提交当前consumer所有消费的vgroup的当前进度 : tmq_commit_sync/tmq_commit_async + - 第二种类型：根据某个topic的某个vgroup的offset提交 : tmq_commit_offset_sync/tmq_commit_offset_async + **参数说明** - msg：消费到的消息结构，如果msg传NULL，提交当前consumer所有消费的vgroup的当前进度 **返回值** - 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 + - `int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId)` **功能说明** - 获取当前消费位置，为消费到的数据位置的下一个位置 - + **返回值** - 消费位置，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 -- `int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` + + - `int32_t tmq_offset_seek(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset)` **功能说明** - 设置 consumer 在某个topic的某个vgroup的 offset位置，开始消费 **返回值** - 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 - -- `int32_t int64_t tmq_get_vgroup_offset(TAOS_RES* res)` + + +- `int64_t tmq_get_vgroup_offset(TAOS_RES* res)` +- `int32_t tmq_get_vgroup_id(TAOS_RES *res)` **功能说明** - - 获取 poll 消费到的数据的起始offset - + - tmq_get_vgroup_offset 获取 poll 消费到的数据的起始offset + - tmq_get_vgroup_id 获取 poll 消费到的数据的所属的vgrou id + **参数说明** - msg：消费到的消息结构 **返回值** - - 消费到的offset，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 - -- `int32_t int32_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics)` + - tmq_get_vgroup_offset 返回值为消费到的offset，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 + - tmq_get_vgroup_id 返回值为消费到的数据所属的vgrou id，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 + + +- `TAOS *tmq_get_connect(tmq_t *tmq)` +- `const char *tmq_get_table_name(TAOS_RES *res)` +- `tmq_res_t tmq_get_res_type(TAOS_RES *res)` +- `const char *tmq_get_topic_name(TAOS_RES *res)` +- `const char *tmq_get_db_name(TAOS_RES *res)` + + tmq_res_t 表示消费到的数据类型，定义如下： + ``` + typedef enum tmq_res_t { + TMQ_RES_INVALID = -1, // invalid + TMQ_RES_DATA = 1, // 数据 + TMQ_RES_TABLE_META = 2, // 元数据 + TMQ_RES_METADATA = 3 // 既有元数据又有数据，即自动建表 + } tmq_res_t; + ``` **功能说明** - - 获取消费者订阅的 topic 列表 - + - tmq_get_connect 创建consumer时，会自动建立链接保存在 tmq_t 结构体中，该接口用户获取 tmq_t 结构体中的链接信息，类似taos_connect + - tmq_get_table_name 获取返回结果所属的的表名 + - tmq_get_res_type 获取返回结果的类型 + - tmq_get_topic_name 获取返回结果所属的topic名 + - tmq_get_db_name 获取返回结果所属的数据库名 + **参数说明** - - topics: 获取的 topic 列表存储在这个结构中，接口内分配内存，需调用tmq_list_destroy释放 + - tmq：tmq_consumer_new 返回的消费者handle + - res：tmq_consumer_poll 返回的消费到的消息 **返回值** - - 错误码，0成功，非0失败，可通过 `char *tmq_err2str(int32_t code)` 函数获取错误信息 + - tmq_get_connect 返回值为tmq_t结构体中的链接连接，非 NULL 正常，NULL 失败 + - tmq_get_table_name 返回值为消费到的数据所属的表名，非 NULL 正常，NULL 失败 + - tmq_get_res_type 返回值为消费到的数据所属的类型，具体见上面 tmq_res_t 的注释说明 + - tmq_get_topic_name 返回值为消费到的数据所属的 topic 名，非 NULL 正常，NULL 失败 + - tmq_get_db_name 返回值为消费到的数据所属的数据库名，非 NULL 正常，NULL 失败 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 5691716ce8..cc057c3b72 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -91,7 +91,7 @@ SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WIN CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); ``` -PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 sutable 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _groupId)。 +PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 SUBTABLE 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 __stableName_groupId)。 注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。 @@ -190,7 +190,7 @@ T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的 TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项指定： -1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 +1. 增量计算，即 IGNORE EXPIRED 0。 2. 直接丢弃，即 IGNORE EXPIRED 1：默认配置，忽略过期数据 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index f0347f424c..6fce985927 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -419,7 +419,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | dnode 支持的最大 vnode 数目 | | 取值范围 | 0-4096 | -| 缺省值 | CPU 核数的 2 倍 | +| 缺省值 | CPU 核数的 2 倍 + 5 | ## 性能调优 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 93b24c9b69..f69e1fd4a8 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.3.0.3 + + + ## 3.3.0.0 diff --git a/include/client/taos.h b/include/client/taos.h index 45dc85f6d9..46e4e7633b 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -282,7 +282,7 @@ typedef enum tmq_res_t { TMQ_RES_INVALID = -1, TMQ_RES_DATA = 1, TMQ_RES_TABLE_META = 2, - TMQ_RES_METADATA = 3, + TMQ_RES_METADATA = 3 } tmq_res_t; typedef struct tmq_topic_assignment { diff --git a/include/common/rsync.h b/include/common/rsync.h index f613a35f48..0840b51793 100644 --- a/include/common/rsync.h +++ b/include/common/rsync.h @@ -13,7 +13,7 @@ extern "C" { void stopRsync(); void startRsync(); -int32_t uploadRsync(const char* id, const char* path); +int32_t uploadByRsync(const char* id, const char* path); int32_t downloadRsync(const char* id, const char* path); int32_t deleteRsync(const char* id); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index d28477ae40..a5a805ff40 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -144,6 +144,7 @@ enum { TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__POLL_DATA_META_RSP, TMQ_MSG_TYPE__WALINFO_RSP, + TMQ_MSG_TYPE__POLL_BATCH_META_RSP, }; enum { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 32cd7bb2ab..197fa125f5 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -274,7 +274,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf, c int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); -bool alreadyAddGroupId(char* ctbName); +bool alreadyAddGroupId(char* ctbName, int64_t groupId); bool isAutoTableName(char* ctbName); void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 17ed732af8..90ee6f7cc0 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -235,6 +235,7 @@ extern int32_t tsMqRebalanceInterval; extern int32_t tsStreamCheckpointInterval; extern float tsSinkDataRate; extern int32_t tsStreamNodeCheckInterval; +extern int32_t tsMaxConcurrentCheckpoint; extern int32_t tsTtlUnit; extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlBatchDropNum; @@ -257,8 +258,10 @@ extern bool tsExperimental; int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); +int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs); int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, - bool tsc, bool isDumpCfg); + bool tsc); void taosCleanupCfg(); int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer); diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 7e7162204e..b707045bd1 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -73,7 +73,7 @@ char *tGetMachineId(); #ifdef TD_ENTERPRISE #define GRANTS_SCHEMA \ static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "version", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ @@ -85,7 +85,7 @@ char *tGetMachineId(); #else #define GRANTS_SCHEMA \ static const SSysDbTableSchema grantsSchema[] = { \ - {.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ + {.name = "version", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, \ diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 29b9e39fb2..0aad976f28 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -158,19 +158,20 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_MAX, } EShowType; -#define TSDB_ALTER_TABLE_ADD_TAG 1 -#define TSDB_ALTER_TABLE_DROP_TAG 2 -#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 -#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 -#define TSDB_ALTER_TABLE_ADD_COLUMN 5 -#define TSDB_ALTER_TABLE_DROP_COLUMN 6 -#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 -#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 -#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 -#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 -#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11 -#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12 -#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13 +#define TSDB_ALTER_TABLE_ADD_TAG 1 +#define TSDB_ALTER_TABLE_DROP_TAG 2 +#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 +#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 +#define TSDB_ALTER_TABLE_ADD_COLUMN 5 +#define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 +#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 +#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 +#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11 +#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12 +#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13 +#define TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION 14 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -2323,10 +2324,10 @@ int32_t tDeserializeSVArbHeartBeatRsp(void* buf, int32_t bufLen, SVArbHeartBeatR void tFreeSVArbHeartBeatRsp(SVArbHeartBeatRsp* pRsp); typedef struct { - char* arbToken; + char* arbToken; int64_t arbTerm; - char* member0Token; - char* member1Token; + char* member0Token; + char* member1Token; } SVArbCheckSyncReq; int32_t tSerializeSVArbCheckSyncReq(void* buf, int32_t bufLen, SVArbCheckSyncReq* pReq); @@ -2346,9 +2347,9 @@ int32_t tDeserializeSVArbCheckSyncRsp(void* buf, int32_t bufLen, SVArbCheckSyncR void tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp* pRsp); typedef struct { - char* arbToken; - int64_t arbTerm; - char* memberToken; + char* arbToken; + int64_t arbTerm; + char* memberToken; } SVArbSetAssignedLeaderReq; int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq); @@ -2368,20 +2369,31 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp* pRsp); typedef struct { int32_t dnodeId; char* token; -} SMArbUpdateGroupReqMember; +} SMArbUpdateGroupMember; typedef struct { - int32_t vgId; - int64_t dbUid; - SMArbUpdateGroupReqMember members[2]; - int8_t isSync; - SMArbUpdateGroupReqMember assignedLeader; - int64_t version; -} SMArbUpdateGroupReq; + int32_t dnodeId; + char* token; + int8_t acked; +} SMArbUpdateGroupAssigned; -int32_t tSerializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq); -int32_t tDeserializeSMArbUpdateGroupReq(void* buf, int32_t bufLen, SMArbUpdateGroupReq* pReq); -void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq* pReq); +typedef struct { + int32_t vgId; + int64_t dbUid; + SMArbUpdateGroupMember members[2]; + int8_t isSync; + int8_t assignedAcked; + SMArbUpdateGroupAssigned assignedLeader; + int64_t version; +} SMArbUpdateGroup; + +typedef struct { + SArray* updateArray; // SMArbUpdateGroup +} SMArbUpdateGroupBatchReq; + +int32_t tSerializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq); +int32_t tDeserializeSMArbUpdateGroupBatchReq(void* buf, int32_t bufLen, SMArbUpdateGroupBatchReq* pReq); +void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq* pReq); typedef struct { char queryStrId[TSDB_QUERY_ID_LEN]; @@ -2707,8 +2719,8 @@ typedef struct { int64_t lastTs; SArray* pVgroupVerList; // 3.3.0.0 - SArray* pCols; // array of SField - int64_t smaId; + SArray* pCols; // array of SField + int64_t smaId; } SCMCreateStreamReq; typedef struct { @@ -2755,6 +2767,7 @@ typedef struct { int32_t autoCommitInterval; int8_t resetOffsetCfg; int8_t enableReplay; + int8_t enableBatchMeta; } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { @@ -2775,6 +2788,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval); tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg); tlen += taosEncodeFixedI8(buf, pReq->enableReplay); + tlen += taosEncodeFixedI8(buf, pReq->enableBatchMeta); return tlen; } @@ -2799,6 +2813,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval); buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg); buf = taosDecodeFixedI8(buf, &pReq->enableReplay); + buf = taosDecodeFixedI8(buf, &pReq->enableBatchMeta); return buf; } @@ -2987,6 +3002,7 @@ typedef struct { int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); int tDecodeSVCreateTbBatchReq(SDecoder* pCoder, SVCreateTbBatchReq* pReq); +void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq* pReq); typedef struct { int32_t code; @@ -3453,23 +3469,24 @@ int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamR int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); void tFreeMDropStreamReq(SMDropStreamReq* pReq); -typedef struct { - char name[TSDB_STREAM_FNAME_LEN]; - int8_t igNotExists; -} SMRecoverStreamReq; - -typedef struct { - int8_t reserved; -} SMRecoverStreamRsp; - typedef struct { int64_t recoverObjUid; int32_t taskId; int32_t hasCheckPoint; } SMVStreamGatherInfoReq; -// int32_t tSerializeSMRecoverStreamReq(void* buf, int32_t bufLen, const SMRecoverStreamReq* pReq); -// int32_t tDeserializeSMRecoverStreamReq(void* buf, int32_t bufLen, SMRecoverStreamReq* pReq); +typedef struct SVUpdateCheckpointInfoReq { + SMsgHead head; + int64_t streamId; + int32_t taskId; + int64_t checkpointId; + int64_t checkpointVer; + int64_t checkpointTs; + int32_t transId; + int8_t dropRelHTask; + int64_t hStreamId; + int64_t hTaskId; +} SVUpdateCheckpointInfoReq; typedef struct { int64_t leftForVer; @@ -3576,7 +3593,7 @@ static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t u pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA; pOffsetVal->uid = uid; pOffsetVal->ts = ts; - if(IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)){ + if (IS_VAR_DATA_TYPE(pOffsetVal->primaryKey.type)) { taosMemoryFree(pOffsetVal->primaryKey.pData); } pOffsetVal->primaryKey = primaryKey; @@ -3676,10 +3693,10 @@ typedef struct { char* ast; int64_t deleteMark; int64_t lastTs; - int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0 + int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0 SArray* pVgroupVerList; int8_t recursiveTsma; - char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma + char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma } SMCreateSmaReq; int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); @@ -3904,11 +3921,12 @@ typedef struct { STqOffsetVal reqOffset; int8_t enableReplay; int8_t sourceExcluded; + int8_t enableBatchMeta; } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); int32_t tDeserializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); -void tDestroySMqPollReq(SMqPollReq *pReq); +void tDestroySMqPollReq(SMqPollReq* pReq); typedef struct { int32_t vgId; @@ -3952,7 +3970,7 @@ typedef struct { int32_t tEncodeMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp); int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); -void tDeleteMqMetaRsp(SMqMetaRsp *pRsp); +void tDeleteMqMetaRsp(SMqMetaRsp* pRsp); #define MQ_DATA_RSP_VERSION 100 @@ -3989,6 +4007,20 @@ int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const void* pRsp); int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, void* pRsp); void tDeleteSTaosxRsp(void* pRsp); +typedef struct SMqBatchMetaRsp { + SMqRspHead head; // not serialize + STqOffsetVal rspOffset; + SArray* batchMetaLen; + SArray* batchMetaReq; + void* pMetaBuff; // not serialize + uint32_t metaBuffLen; // not serialize +} SMqBatchMetaRsp; + +int32_t tEncodeMqBatchMetaRsp(SEncoder* pEncoder, const SMqBatchMetaRsp* pRsp); +int32_t tDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp); +int32_t tSemiDecodeMqBatchMetaRsp(SDecoder* pDecoder, SMqBatchMetaRsp* pRsp); +void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp* pRsp); + typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; @@ -4299,9 +4331,9 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp); void tFreeSViewMetaRsp(SViewMetaRsp* pRsp); typedef struct { - char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name - bool fetchingWithTsmaName; // if we are fetching with tsma name -}STableTSMAInfoReq; + char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name + bool fetchingWithTsmaName; // if we are fetching with tsma name +} STableTSMAInfoReq; int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq); int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq); @@ -4324,15 +4356,15 @@ typedef struct { int32_t version; int64_t interval; int8_t unit; - SArray* pFuncs; // SArray - SArray* pTags; // SArray - SArray* pUsedCols; // SArray + SArray* pFuncs; // SArray + SArray* pTags; // SArray + SArray* pUsedCols; // SArray char* ast; int64_t streamUid; int64_t reqTs; int64_t rspTs; - int64_t delayDuration; // ms + int64_t delayDuration; // ms bool fillHistoryFinished; } STableTSMAInfo; @@ -4343,10 +4375,10 @@ void tFreeTableTSMAInfo(void* p); void tFreeAndClearTableTSMAInfo(void* p); void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp); -#define STSMAHbRsp STableTSMAInfoRsp -#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp +#define STSMAHbRsp STableTSMAInfoRsp +#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp #define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp -#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp +#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp typedef struct SStreamProgressReq { int64_t streamId; @@ -4372,7 +4404,7 @@ int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgres typedef struct SDropCtbWithTsmaSingleVgReq { SVgroupInfo vgInfo; - SArray* pTbs; // SVDropTbReq + SArray* pTbs; // SVDropTbReq } SMDropTbReqsOnSingleVg; int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq); @@ -4380,7 +4412,7 @@ int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* void tFreeSMDropTbReqOnSingleVg(void* p); typedef struct SDropTbsReq { - SArray* pVgReqs; // SMDropTbReqsOnSingleVg + SArray* pVgReqs; // SMDropTbReqsOnSingleVg } SMDropTbsReq; int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a5a3bd5ee0..6e1980c0af 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -224,13 +224,14 @@ TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ENCRYPT_KEY, "create-encrypt-key", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_UNUSED2, "mnd-unused2", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TSMA, "create-tsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TSMA, "drop-tsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STB_DROP, "drop-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_TSMA, "get-table-tsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_TSMA, "get-tsma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 @@ -275,7 +276,6 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) @@ -286,6 +286,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) @@ -310,7 +311,7 @@ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_UPDATE_CHKPT, "stream-update-chkptinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) //1035 1036 TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT_READY, "stream-checkpoint-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL) @@ -318,9 +319,12 @@ TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_PAUSE, "stream-task-pause", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED, "stream-unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE_TRIGGER, "stream-retri-trigger", NULL, NULL) + + TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 @@ -367,8 +371,9 @@ TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_UPDATE, "vnode-stream-update", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_UNUSED, "vnd-stream-unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 @@ -388,7 +393,8 @@ TD_NEW_MSG_SEG(TDMT_MND_ARB_MSG) //9 << 8 TD_DEF_MSG_TYPE(TDMT_MND_ARB_HEARTBEAT_TIMER, "mnd-arb-hb-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ARB_CHECK_SYNC_TIMER, "mnd-arb-check-sync-tmr", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP, "mnd-arb-update-group", NULL, NULL) // no longer used + TD_DEF_MSG_TYPE(TDMT_MND_ARB_UPDATE_GROUP_BATCH, "mnd-arb-update-group-batch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ARB_MAX_MSG, "mnd-arb-max", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_ARB_MSG) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index e619edf972..0c15fd3b9d 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -16,381 +16,381 @@ #ifndef _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_ -#define TK_OR 1 -#define TK_AND 2 -#define TK_UNION 3 -#define TK_ALL 4 -#define TK_MINUS 5 -#define TK_EXCEPT 6 -#define TK_INTERSECT 7 -#define TK_NK_BITAND 8 -#define TK_NK_BITOR 9 -#define TK_NK_LSHIFT 10 -#define TK_NK_RSHIFT 11 -#define TK_NK_PLUS 12 -#define TK_NK_MINUS 13 -#define TK_NK_STAR 14 -#define TK_NK_SLASH 15 -#define TK_NK_REM 16 -#define TK_NK_CONCAT 17 -#define TK_CREATE 18 -#define TK_ACCOUNT 19 -#define TK_NK_ID 20 -#define TK_PASS 21 -#define TK_NK_STRING 22 -#define TK_ALTER 23 -#define TK_PPS 24 -#define TK_TSERIES 25 -#define TK_STORAGE 26 -#define TK_STREAMS 27 -#define TK_QTIME 28 -#define TK_DBS 29 -#define TK_USERS 30 -#define TK_CONNS 31 -#define TK_STATE 32 -#define TK_NK_COMMA 33 -#define TK_HOST 34 -#define TK_USER 35 -#define TK_ENABLE 36 -#define TK_NK_INTEGER 37 -#define TK_SYSINFO 38 -#define TK_ADD 39 -#define TK_DROP 40 -#define TK_GRANT 41 -#define TK_ON 42 -#define TK_TO 43 -#define TK_REVOKE 44 -#define TK_FROM 45 -#define TK_SUBSCRIBE 46 -#define TK_READ 47 -#define TK_WRITE 48 -#define TK_NK_DOT 49 -#define TK_WITH 50 -#define TK_ENCRYPT_KEY 51 -#define TK_DNODE 52 -#define TK_PORT 53 -#define TK_DNODES 54 -#define TK_RESTORE 55 -#define TK_NK_IPTOKEN 56 -#define TK_FORCE 57 -#define TK_UNSAFE 58 -#define TK_CLUSTER 59 -#define TK_LOCAL 60 -#define TK_QNODE 61 -#define TK_BNODE 62 -#define TK_SNODE 63 -#define TK_MNODE 64 -#define TK_VNODE 65 -#define TK_DATABASE 66 -#define TK_USE 67 -#define TK_FLUSH 68 -#define TK_TRIM 69 -#define TK_S3MIGRATE 70 -#define TK_COMPACT 71 -#define TK_IF 72 -#define TK_NOT 73 -#define TK_EXISTS 74 -#define TK_BUFFER 75 -#define TK_CACHEMODEL 76 -#define TK_CACHESIZE 77 -#define TK_COMP 78 -#define TK_DURATION 79 -#define TK_NK_VARIABLE 80 -#define TK_MAXROWS 81 -#define TK_MINROWS 82 -#define TK_KEEP 83 -#define TK_PAGES 84 -#define TK_PAGESIZE 85 -#define TK_TSDB_PAGESIZE 86 -#define TK_PRECISION 87 -#define TK_REPLICA 88 -#define TK_VGROUPS 89 -#define TK_SINGLE_STABLE 90 -#define TK_RETENTIONS 91 -#define TK_SCHEMALESS 92 -#define TK_WAL_LEVEL 93 -#define TK_WAL_FSYNC_PERIOD 94 -#define TK_WAL_RETENTION_PERIOD 95 -#define TK_WAL_RETENTION_SIZE 96 -#define TK_WAL_ROLL_PERIOD 97 -#define TK_WAL_SEGMENT_SIZE 98 -#define TK_STT_TRIGGER 99 -#define TK_TABLE_PREFIX 100 -#define TK_TABLE_SUFFIX 101 -#define TK_S3_CHUNKSIZE 102 -#define TK_S3_KEEPLOCAL 103 -#define TK_S3_COMPACT 104 -#define TK_KEEP_TIME_OFFSET 105 -#define TK_ENCRYPT_ALGORITHM 106 -#define TK_NK_COLON 107 -#define TK_BWLIMIT 108 -#define TK_START 109 -#define TK_TIMESTAMP 110 -#define TK_END 111 -#define TK_TABLE 112 -#define TK_NK_LP 113 -#define TK_NK_RP 114 -#define TK_STABLE 115 -#define TK_COLUMN 116 -#define TK_MODIFY 117 -#define TK_RENAME 118 -#define TK_TAG 119 -#define TK_SET 120 -#define TK_NK_EQ 121 -#define TK_USING 122 -#define TK_TAGS 123 -#define TK_BOOL 124 -#define TK_TINYINT 125 -#define TK_SMALLINT 126 -#define TK_INT 127 -#define TK_INTEGER 128 -#define TK_BIGINT 129 -#define TK_FLOAT 130 -#define TK_DOUBLE 131 -#define TK_BINARY 132 -#define TK_NCHAR 133 -#define TK_UNSIGNED 134 -#define TK_JSON 135 -#define TK_VARCHAR 136 -#define TK_MEDIUMBLOB 137 -#define TK_BLOB 138 -#define TK_VARBINARY 139 -#define TK_GEOMETRY 140 -#define TK_DECIMAL 141 -#define TK_COMMENT 142 -#define TK_MAX_DELAY 143 -#define TK_WATERMARK 144 -#define TK_ROLLUP 145 -#define TK_TTL 146 -#define TK_SMA 147 -#define TK_DELETE_MARK 148 -#define TK_FIRST 149 -#define TK_LAST 150 -#define TK_SHOW 151 -#define TK_PRIVILEGES 152 -#define TK_DATABASES 153 -#define TK_TABLES 154 -#define TK_STABLES 155 -#define TK_MNODES 156 -#define TK_QNODES 157 -#define TK_ARBGROUPS 158 -#define TK_FUNCTIONS 159 -#define TK_INDEXES 160 -#define TK_ACCOUNTS 161 -#define TK_APPS 162 -#define TK_CONNECTIONS 163 -#define TK_LICENCES 164 -#define TK_GRANTS 165 -#define TK_FULL 166 -#define TK_LOGS 167 -#define TK_MACHINES 168 -#define TK_ENCRYPTIONS 169 -#define TK_QUERIES 170 -#define TK_SCORES 171 -#define TK_TOPICS 172 -#define TK_VARIABLES 173 -#define TK_BNODES 174 -#define TK_SNODES 175 -#define TK_TRANSACTIONS 176 -#define TK_DISTRIBUTED 177 -#define TK_CONSUMERS 178 -#define TK_SUBSCRIPTIONS 179 -#define TK_VNODES 180 -#define TK_ALIVE 181 -#define TK_VIEWS 182 -#define TK_VIEW 183 -#define TK_COMPACTS 184 -#define TK_NORMAL 185 -#define TK_CHILD 186 -#define TK_LIKE 187 -#define TK_TBNAME 188 -#define TK_QTAGS 189 -#define TK_AS 190 -#define TK_SYSTEM 191 -#define TK_TSMA 192 -#define TK_INTERVAL 193 -#define TK_RECURSIVE 194 -#define TK_TSMAS 195 -#define TK_FUNCTION 196 -#define TK_INDEX 197 -#define TK_COUNT 198 -#define TK_LAST_ROW 199 -#define TK_META 200 -#define TK_ONLY 201 -#define TK_TOPIC 202 -#define TK_CONSUMER 203 -#define TK_GROUP 204 -#define TK_DESC 205 -#define TK_DESCRIBE 206 -#define TK_RESET 207 -#define TK_QUERY 208 -#define TK_CACHE 209 -#define TK_EXPLAIN 210 -#define TK_ANALYZE 211 -#define TK_VERBOSE 212 -#define TK_NK_BOOL 213 -#define TK_RATIO 214 -#define TK_NK_FLOAT 215 -#define TK_OUTPUTTYPE 216 -#define TK_AGGREGATE 217 -#define TK_BUFSIZE 218 -#define TK_LANGUAGE 219 -#define TK_REPLACE 220 -#define TK_STREAM 221 -#define TK_INTO 222 -#define TK_PAUSE 223 -#define TK_RESUME 224 -#define TK_PRIMARY 225 -#define TK_KEY 226 -#define TK_TRIGGER 227 -#define TK_AT_ONCE 228 -#define TK_WINDOW_CLOSE 229 -#define TK_IGNORE 230 -#define TK_EXPIRED 231 -#define TK_FILL_HISTORY 232 -#define TK_UPDATE 233 -#define TK_SUBTABLE 234 -#define TK_UNTREATED 235 -#define TK_KILL 236 -#define TK_CONNECTION 237 -#define TK_TRANSACTION 238 -#define TK_BALANCE 239 -#define TK_VGROUP 240 -#define TK_LEADER 241 -#define TK_MERGE 242 -#define TK_REDISTRIBUTE 243 -#define TK_SPLIT 244 -#define TK_DELETE 245 -#define TK_INSERT 246 -#define TK_NK_BIN 247 -#define TK_NK_HEX 248 -#define TK_NULL 249 -#define TK_NK_QUESTION 250 -#define TK_NK_ALIAS 251 -#define TK_NK_ARROW 252 -#define TK_ROWTS 253 -#define TK_QSTART 254 -#define TK_QEND 255 -#define TK_QDURATION 256 -#define TK_WSTART 257 -#define TK_WEND 258 -#define TK_WDURATION 259 -#define TK_IROWTS 260 -#define TK_ISFILLED 261 -#define TK_CAST 262 -#define TK_NOW 263 -#define TK_TODAY 264 -#define TK_TIMEZONE 265 -#define TK_CLIENT_VERSION 266 -#define TK_SERVER_VERSION 267 -#define TK_SERVER_STATUS 268 -#define TK_CURRENT_USER 269 -#define TK_CASE 270 -#define TK_WHEN 271 -#define TK_THEN 272 -#define TK_ELSE 273 -#define TK_BETWEEN 274 -#define TK_IS 275 -#define TK_NK_LT 276 -#define TK_NK_GT 277 -#define TK_NK_LE 278 -#define TK_NK_GE 279 -#define TK_NK_NE 280 -#define TK_MATCH 281 -#define TK_NMATCH 282 -#define TK_CONTAINS 283 -#define TK_IN 284 -#define TK_JOIN 285 -#define TK_INNER 286 -#define TK_LEFT 287 -#define TK_RIGHT 288 -#define TK_OUTER 289 -#define TK_SEMI 290 -#define TK_ANTI 291 -#define TK_ASOF 292 -#define TK_WINDOW 293 -#define TK_WINDOW_OFFSET 294 -#define TK_JLIMIT 295 -#define TK_SELECT 296 -#define TK_NK_HINT 297 -#define TK_DISTINCT 298 -#define TK_WHERE 299 -#define TK_PARTITION 300 -#define TK_BY 301 -#define TK_SESSION 302 -#define TK_STATE_WINDOW 303 -#define TK_EVENT_WINDOW 304 -#define TK_COUNT_WINDOW 305 -#define TK_SLIDING 306 -#define TK_FILL 307 -#define TK_VALUE 308 -#define TK_VALUE_F 309 -#define TK_NONE 310 -#define TK_PREV 311 -#define TK_NULL_F 312 -#define TK_LINEAR 313 -#define TK_NEXT 314 -#define TK_HAVING 315 -#define TK_RANGE 316 -#define TK_EVERY 317 -#define TK_ORDER 318 -#define TK_SLIMIT 319 -#define TK_SOFFSET 320 -#define TK_LIMIT 321 -#define TK_OFFSET 322 -#define TK_ASC 323 -#define TK_NULLS 324 -#define TK_ABORT 325 -#define TK_AFTER 326 -#define TK_ATTACH 327 -#define TK_BEFORE 328 -#define TK_BEGIN 329 -#define TK_BITAND 330 -#define TK_BITNOT 331 -#define TK_BITOR 332 -#define TK_BLOCKS 333 -#define TK_CHANGE 334 -#define TK_COMMA 335 -#define TK_CONCAT 336 -#define TK_CONFLICT 337 -#define TK_COPY 338 -#define TK_DEFERRED 339 -#define TK_DELIMITERS 340 -#define TK_DETACH 341 -#define TK_DIVIDE 342 -#define TK_DOT 343 -#define TK_EACH 344 -#define TK_FAIL 345 -#define TK_FILE 346 -#define TK_FOR 347 -#define TK_GLOB 348 -#define TK_ID 349 -#define TK_IMMEDIATE 350 -#define TK_IMPORT 351 -#define TK_INITIALLY 352 -#define TK_INSTEAD 353 -#define TK_ISNULL 354 -#define TK_MODULES 355 -#define TK_NK_BITNOT 356 -#define TK_NK_SEMI 357 -#define TK_NOTNULL 358 -#define TK_OF 359 -#define TK_PLUS 360 -#define TK_PRIVILEGE 361 -#define TK_RAISE 362 -#define TK_RESTRICT 363 -#define TK_ROW 364 -#define TK_STAR 365 -#define TK_STATEMENT 366 -#define TK_STRICT 367 -#define TK_STRING 368 -#define TK_TIMES 369 -#define TK_VALUES 370 -#define TK_VARIABLE 371 -#define TK_WAL 372 -#define TK_ENCODE 373 -#define TK_COMPRESS 374 -#define TK_LEVEL 375 +#define TK_OR 1 +#define TK_AND 2 +#define TK_UNION 3 +#define TK_ALL 4 +#define TK_MINUS 5 +#define TK_EXCEPT 6 +#define TK_INTERSECT 7 +#define TK_NK_BITAND 8 +#define TK_NK_BITOR 9 +#define TK_NK_LSHIFT 10 +#define TK_NK_RSHIFT 11 +#define TK_NK_PLUS 12 +#define TK_NK_MINUS 13 +#define TK_NK_STAR 14 +#define TK_NK_SLASH 15 +#define TK_NK_REM 16 +#define TK_NK_CONCAT 17 +#define TK_CREATE 18 +#define TK_ACCOUNT 19 +#define TK_NK_ID 20 +#define TK_PASS 21 +#define TK_NK_STRING 22 +#define TK_ALTER 23 +#define TK_PPS 24 +#define TK_TSERIES 25 +#define TK_STORAGE 26 +#define TK_STREAMS 27 +#define TK_QTIME 28 +#define TK_DBS 29 +#define TK_USERS 30 +#define TK_CONNS 31 +#define TK_STATE 32 +#define TK_NK_COMMA 33 +#define TK_HOST 34 +#define TK_USER 35 +#define TK_ENABLE 36 +#define TK_NK_INTEGER 37 +#define TK_SYSINFO 38 +#define TK_ADD 39 +#define TK_DROP 40 +#define TK_GRANT 41 +#define TK_ON 42 +#define TK_TO 43 +#define TK_REVOKE 44 +#define TK_FROM 45 +#define TK_SUBSCRIBE 46 +#define TK_READ 47 +#define TK_WRITE 48 +#define TK_NK_DOT 49 +#define TK_WITH 50 +#define TK_ENCRYPT_KEY 51 +#define TK_DNODE 52 +#define TK_PORT 53 +#define TK_DNODES 54 +#define TK_RESTORE 55 +#define TK_NK_IPTOKEN 56 +#define TK_FORCE 57 +#define TK_UNSAFE 58 +#define TK_CLUSTER 59 +#define TK_LOCAL 60 +#define TK_QNODE 61 +#define TK_BNODE 62 +#define TK_SNODE 63 +#define TK_MNODE 64 +#define TK_VNODE 65 +#define TK_DATABASE 66 +#define TK_USE 67 +#define TK_FLUSH 68 +#define TK_TRIM 69 +#define TK_S3MIGRATE 70 +#define TK_COMPACT 71 +#define TK_IF 72 +#define TK_NOT 73 +#define TK_EXISTS 74 +#define TK_BUFFER 75 +#define TK_CACHEMODEL 76 +#define TK_CACHESIZE 77 +#define TK_COMP 78 +#define TK_DURATION 79 +#define TK_NK_VARIABLE 80 +#define TK_MAXROWS 81 +#define TK_MINROWS 82 +#define TK_KEEP 83 +#define TK_PAGES 84 +#define TK_PAGESIZE 85 +#define TK_TSDB_PAGESIZE 86 +#define TK_PRECISION 87 +#define TK_REPLICA 88 +#define TK_VGROUPS 89 +#define TK_SINGLE_STABLE 90 +#define TK_RETENTIONS 91 +#define TK_SCHEMALESS 92 +#define TK_WAL_LEVEL 93 +#define TK_WAL_FSYNC_PERIOD 94 +#define TK_WAL_RETENTION_PERIOD 95 +#define TK_WAL_RETENTION_SIZE 96 +#define TK_WAL_ROLL_PERIOD 97 +#define TK_WAL_SEGMENT_SIZE 98 +#define TK_STT_TRIGGER 99 +#define TK_TABLE_PREFIX 100 +#define TK_TABLE_SUFFIX 101 +#define TK_S3_CHUNKSIZE 102 +#define TK_S3_KEEPLOCAL 103 +#define TK_S3_COMPACT 104 +#define TK_KEEP_TIME_OFFSET 105 +#define TK_ENCRYPT_ALGORITHM 106 +#define TK_NK_COLON 107 +#define TK_BWLIMIT 108 +#define TK_START 109 +#define TK_TIMESTAMP 110 +#define TK_END 111 +#define TK_TABLE 112 +#define TK_NK_LP 113 +#define TK_NK_RP 114 +#define TK_STABLE 115 +#define TK_COLUMN 116 +#define TK_MODIFY 117 +#define TK_RENAME 118 +#define TK_TAG 119 +#define TK_SET 120 +#define TK_NK_EQ 121 +#define TK_USING 122 +#define TK_TAGS 123 +#define TK_BOOL 124 +#define TK_TINYINT 125 +#define TK_SMALLINT 126 +#define TK_INT 127 +#define TK_INTEGER 128 +#define TK_BIGINT 129 +#define TK_FLOAT 130 +#define TK_DOUBLE 131 +#define TK_BINARY 132 +#define TK_NCHAR 133 +#define TK_UNSIGNED 134 +#define TK_JSON 135 +#define TK_VARCHAR 136 +#define TK_MEDIUMBLOB 137 +#define TK_BLOB 138 +#define TK_VARBINARY 139 +#define TK_GEOMETRY 140 +#define TK_DECIMAL 141 +#define TK_COMMENT 142 +#define TK_MAX_DELAY 143 +#define TK_WATERMARK 144 +#define TK_ROLLUP 145 +#define TK_TTL 146 +#define TK_SMA 147 +#define TK_DELETE_MARK 148 +#define TK_FIRST 149 +#define TK_LAST 150 +#define TK_SHOW 151 +#define TK_PRIVILEGES 152 +#define TK_DATABASES 153 +#define TK_TABLES 154 +#define TK_STABLES 155 +#define TK_MNODES 156 +#define TK_QNODES 157 +#define TK_ARBGROUPS 158 +#define TK_FUNCTIONS 159 +#define TK_INDEXES 160 +#define TK_ACCOUNTS 161 +#define TK_APPS 162 +#define TK_CONNECTIONS 163 +#define TK_LICENCES 164 +#define TK_GRANTS 165 +#define TK_FULL 166 +#define TK_LOGS 167 +#define TK_MACHINES 168 +#define TK_ENCRYPTIONS 169 +#define TK_QUERIES 170 +#define TK_SCORES 171 +#define TK_TOPICS 172 +#define TK_VARIABLES 173 +#define TK_BNODES 174 +#define TK_SNODES 175 +#define TK_TRANSACTIONS 176 +#define TK_DISTRIBUTED 177 +#define TK_CONSUMERS 178 +#define TK_SUBSCRIPTIONS 179 +#define TK_VNODES 180 +#define TK_ALIVE 181 +#define TK_VIEWS 182 +#define TK_VIEW 183 +#define TK_COMPACTS 184 +#define TK_NORMAL 185 +#define TK_CHILD 186 +#define TK_LIKE 187 +#define TK_TBNAME 188 +#define TK_QTAGS 189 +#define TK_AS 190 +#define TK_SYSTEM 191 +#define TK_TSMA 192 +#define TK_INTERVAL 193 +#define TK_RECURSIVE 194 +#define TK_TSMAS 195 +#define TK_FUNCTION 196 +#define TK_INDEX 197 +#define TK_COUNT 198 +#define TK_LAST_ROW 199 +#define TK_META 200 +#define TK_ONLY 201 +#define TK_TOPIC 202 +#define TK_CONSUMER 203 +#define TK_GROUP 204 +#define TK_DESC 205 +#define TK_DESCRIBE 206 +#define TK_RESET 207 +#define TK_QUERY 208 +#define TK_CACHE 209 +#define TK_EXPLAIN 210 +#define TK_ANALYZE 211 +#define TK_VERBOSE 212 +#define TK_NK_BOOL 213 +#define TK_RATIO 214 +#define TK_NK_FLOAT 215 +#define TK_OUTPUTTYPE 216 +#define TK_AGGREGATE 217 +#define TK_BUFSIZE 218 +#define TK_LANGUAGE 219 +#define TK_REPLACE 220 +#define TK_STREAM 221 +#define TK_INTO 222 +#define TK_PAUSE 223 +#define TK_RESUME 224 +#define TK_PRIMARY 225 +#define TK_KEY 226 +#define TK_TRIGGER 227 +#define TK_AT_ONCE 228 +#define TK_WINDOW_CLOSE 229 +#define TK_IGNORE 230 +#define TK_EXPIRED 231 +#define TK_FILL_HISTORY 232 +#define TK_UPDATE 233 +#define TK_SUBTABLE 234 +#define TK_UNTREATED 235 +#define TK_KILL 236 +#define TK_CONNECTION 237 +#define TK_TRANSACTION 238 +#define TK_BALANCE 239 +#define TK_VGROUP 240 +#define TK_LEADER 241 +#define TK_MERGE 242 +#define TK_REDISTRIBUTE 243 +#define TK_SPLIT 244 +#define TK_DELETE 245 +#define TK_INSERT 246 +#define TK_NK_BIN 247 +#define TK_NK_HEX 248 +#define TK_NULL 249 +#define TK_NK_QUESTION 250 +#define TK_NK_ALIAS 251 +#define TK_NK_ARROW 252 +#define TK_ROWTS 253 +#define TK_QSTART 254 +#define TK_QEND 255 +#define TK_QDURATION 256 +#define TK_WSTART 257 +#define TK_WEND 258 +#define TK_WDURATION 259 +#define TK_IROWTS 260 +#define TK_ISFILLED 261 +#define TK_CAST 262 +#define TK_NOW 263 +#define TK_TODAY 264 +#define TK_TIMEZONE 265 +#define TK_CLIENT_VERSION 266 +#define TK_SERVER_VERSION 267 +#define TK_SERVER_STATUS 268 +#define TK_CURRENT_USER 269 +#define TK_CASE 270 +#define TK_WHEN 271 +#define TK_THEN 272 +#define TK_ELSE 273 +#define TK_BETWEEN 274 +#define TK_IS 275 +#define TK_NK_LT 276 +#define TK_NK_GT 277 +#define TK_NK_LE 278 +#define TK_NK_GE 279 +#define TK_NK_NE 280 +#define TK_MATCH 281 +#define TK_NMATCH 282 +#define TK_CONTAINS 283 +#define TK_IN 284 +#define TK_JOIN 285 +#define TK_INNER 286 +#define TK_LEFT 287 +#define TK_RIGHT 288 +#define TK_OUTER 289 +#define TK_SEMI 290 +#define TK_ANTI 291 +#define TK_ASOF 292 +#define TK_WINDOW 293 +#define TK_WINDOW_OFFSET 294 +#define TK_JLIMIT 295 +#define TK_SELECT 296 +#define TK_NK_HINT 297 +#define TK_DISTINCT 298 +#define TK_WHERE 299 +#define TK_PARTITION 300 +#define TK_BY 301 +#define TK_SESSION 302 +#define TK_STATE_WINDOW 303 +#define TK_EVENT_WINDOW 304 +#define TK_COUNT_WINDOW 305 +#define TK_SLIDING 306 +#define TK_FILL 307 +#define TK_VALUE 308 +#define TK_VALUE_F 309 +#define TK_NONE 310 +#define TK_PREV 311 +#define TK_NULL_F 312 +#define TK_LINEAR 313 +#define TK_NEXT 314 +#define TK_HAVING 315 +#define TK_RANGE 316 +#define TK_EVERY 317 +#define TK_ORDER 318 +#define TK_SLIMIT 319 +#define TK_SOFFSET 320 +#define TK_LIMIT 321 +#define TK_OFFSET 322 +#define TK_ASC 323 +#define TK_NULLS 324 +#define TK_ABORT 325 +#define TK_AFTER 326 +#define TK_ATTACH 327 +#define TK_BEFORE 328 +#define TK_BEGIN 329 +#define TK_BITAND 330 +#define TK_BITNOT 331 +#define TK_BITOR 332 +#define TK_BLOCKS 333 +#define TK_CHANGE 334 +#define TK_COMMA 335 +#define TK_CONCAT 336 +#define TK_CONFLICT 337 +#define TK_COPY 338 +#define TK_DEFERRED 339 +#define TK_DELIMITERS 340 +#define TK_DETACH 341 +#define TK_DIVIDE 342 +#define TK_DOT 343 +#define TK_EACH 344 +#define TK_FAIL 345 +#define TK_FILE 346 +#define TK_FOR 347 +#define TK_GLOB 348 +#define TK_ID 349 +#define TK_IMMEDIATE 350 +#define TK_IMPORT 351 +#define TK_INITIALLY 352 +#define TK_INSTEAD 353 +#define TK_ISNULL 354 +#define TK_MODULES 355 +#define TK_NK_BITNOT 356 +#define TK_NK_SEMI 357 +#define TK_NOTNULL 358 +#define TK_OF 359 +#define TK_PLUS 360 +#define TK_PRIVILEGE 361 +#define TK_RAISE 362 +#define TK_RESTRICT 363 +#define TK_ROW 364 +#define TK_STAR 365 +#define TK_STATEMENT 366 +#define TK_STRICT 367 +#define TK_STRING 368 +#define TK_TIMES 369 +#define TK_VALUES 370 +#define TK_VARIABLE 371 +#define TK_WAL 372 +#define TK_ENCODE 373 +#define TK_COMPRESS 374 +#define TK_LEVEL 375 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 82823e3f57..15cb6d59aa 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -44,6 +44,11 @@ int32_t dmRun(); */ void dmStop(); +/** + * for tests + */ +bool dmReadyForTest(); + #ifdef __cplusplus } #endif diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 108e6f18a6..fe96fe1117 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -101,6 +101,8 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr */ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +int64_t mndGetRoleTimeMs(SMnode *pMnode); + /** * @brief Process the rpc, sync request. * diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index ce04ec6953..0076d79312 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -36,10 +36,13 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTasksGetTotalNum(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); +int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); -int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode); +int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta); void tqSetRestoreVersionInfo(SStreamTask* pTask); #endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2774fce33b..78a977c86f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -202,7 +202,7 @@ void qStreamSetSourceExcluded(qTaskInfo_t tinfo, int8_t sourceExcluded); void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); -SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); +SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 2d507ab6bb..330ba31c65 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -410,7 +410,7 @@ typedef struct SStateStore { void (*streamFileStateClear)(struct SStreamFileState* pFileState); bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); - SStreamState* (*streamStateOpen)(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); + SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void (*streamStateClose)(SStreamState* pState, bool remove); int32_t (*streamStateBegin)(SStreamState* pState); int32_t (*streamStateCommit)(SStreamState* pState); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 0afda2e160..87bbe21133 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -150,6 +150,7 @@ typedef struct SBackendCfWrapper { int64_t backendId; char idstr[64]; } SBackendCfWrapper; + typedef struct STdbState { SBackendCfWrapper *pBackendCfWrapper; int64_t backendCfWrapperId; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 31091928c9..d0bf2e2dd4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -104,6 +104,8 @@ typedef struct SNodeAllocator SNodeAllocator; int32_t nodesInitAllocatorSet(); void nodesDestroyAllocatorSet(); int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAllocatorId); +int32_t nodesSimAcquireAllocator(int64_t allocatorId); +int32_t nodesSimReleaseAllocator(int64_t allocatorId); int32_t nodesAcquireAllocator(int64_t allocatorId); int32_t nodesReleaseAllocator(int64_t allocatorId); int64_t nodesMakeAllocatorWeakRef(int64_t allocatorId); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 18bc24d612..a691433ee6 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -183,6 +183,7 @@ typedef struct SProjectLogicNode { char stmtName[TSDB_TABLE_NAME_LEN]; bool ignoreGroupId; bool inputIgnoreGroup; + bool isSetOpProj; } SProjectLogicNode; typedef struct SIndefRowsFuncLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 5b889171b3..befd6afce9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -415,6 +415,7 @@ typedef struct SSelectStmt { int32_t returnRows; // EFuncReturnRows ETimeLineMode timeLineCurMode; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool isEmptyResult; bool isSubquery; bool hasAggFuncs; @@ -453,6 +454,7 @@ typedef struct SSetOperator { char stmtName[TSDB_TABLE_NAME_LEN]; uint8_t precision; ETimeLineMode timeLineResMode; + bool timeLineFromOrderBy; bool joinContains; } SSetOperator; diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 7813b2cc9a..ae5a733ae9 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -29,7 +29,7 @@ extern "C" { #include "storageapi.h" -SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h index 5436442284..91bfc6afc8 100644 --- a/include/libs/stream/streammsg.h +++ b/include/libs/stream/streammsg.h @@ -22,17 +22,17 @@ extern "C" { #endif -typedef struct SStreamChildEpInfo { +typedef struct SStreamUpstreamEpInfo { int32_t nodeId; int32_t childId; int32_t taskId; SEpSet epSet; bool dataAllowed; // denote if the data from this upstream task is allowed to put into inputQ, not serialize it int64_t stage; // upstream task stage value, to denote if the upstream node has restart/replica changed/transfer -} SStreamChildEpInfo; +} SStreamUpstreamEpInfo; -int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); -int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); +int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo); +int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo); // mndTrigger: denote if this checkpoint is triggered by mnode or as requested from tasks when transfer-state finished typedef struct { @@ -171,6 +171,25 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp); int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pRsp); void tCleanupStreamHbMsg(SStreamHbMsg* pMsg); +typedef struct SRetrieveChkptTriggerReq { + SMsgHead head; + int64_t streamId; + int64_t checkpointId; + int32_t upstreamNodeId; + int32_t upstreamTaskId; + int32_t downstreamNodeId; + int64_t downstreamTaskId; +} SRetrieveChkptTriggerReq; + +typedef struct SCheckpointTriggerRsp { + int64_t streamId; + int64_t checkpointId; + int32_t upstreamTaskId; + int32_t taskId; + int32_t transId; + int32_t rspCode; +} SCheckpointTriggerRsp; + typedef struct { SMsgHead head; int64_t streamId; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 3c5d6d6e4c..d07a302920 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -58,11 +58,14 @@ extern "C" { #define STREAM_EXEC_T_STOP_ALL_TASKS (-5) #define STREAM_EXEC_T_RESUME_TASK (-6) #define STREAM_EXEC_T_ADD_FAILED_TASK (-7) +// the load and start stream task should be executed after snode has started successfully, since the load of stream +// tasks may incur the download of checkpoint data from remote, which may consume significant network and CPU resources. -typedef struct SStreamTask SStreamTask; -typedef struct SStreamQueue SStreamQueue; -typedef struct SStreamTaskSM SStreamTaskSM; -typedef struct SStreamQueueItem SStreamQueueItem; +typedef struct SStreamTask SStreamTask; +typedef struct SStreamQueue SStreamQueue; +typedef struct SStreamTaskSM SStreamTaskSM; +typedef struct SStreamQueueItem SStreamQueueItem; +typedef struct SActiveCheckpointInfo SActiveCheckpointInfo; #define SSTREAM_TASK_VER 4 #define SSTREAM_TASK_INCOMPATIBLE_VER 1 @@ -152,8 +155,6 @@ typedef enum EStreamTaskEvent { TASK_EVENT_DROPPING = 0xA, } EStreamTaskEvent; -typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); - typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data); typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); @@ -270,13 +271,8 @@ typedef struct SCheckpointInfo { int64_t checkpointTime; // latest checkpoint time int64_t processedVer; int64_t nextProcessVer; // current offset in WAL, not serialize it - int64_t failedId; // record the latest failed checkpoint id - int64_t checkpointingId; - int32_t downstreamAlignNum; - int32_t numOfNotReady; - bool dispatchCheckpointTrigger; + SActiveCheckpointInfo* pActiveInfo; int64_t msgVer; - int32_t transId; } SCheckpointInfo; typedef struct SStreamStatus { @@ -290,6 +286,7 @@ typedef struct SStreamStatus { int64_t lastExecTs; // last exec time stamp int32_t inScanHistorySentinel; bool appendTranstateBlock; // has append the transfer state data block already + bool removeBackendFiles; // remove backend files on disk when free stream tasks } SStreamStatus; typedef struct SDataRange { @@ -305,7 +302,7 @@ typedef struct SSTaskBasicInfo { int32_t totalLevel; int8_t taskLevel; int8_t fillHistory; // is fill history task or not - int64_t triggerParam; // in msec + int64_t delaySchedParam; // in msec } SSTaskBasicInfo; typedef struct SStreamRetrieveReq SStreamRetrieveReq; @@ -322,7 +319,8 @@ typedef struct SDispatchMsgInfo { int32_t retryCount; // retry send data count int64_t startTs; // dispatch start time, record total elapsed time for dispatch SArray* pRetryList; // current dispatch successfully completed node of downstream - void* pTimer; // used to dispatch data after a given time duration + void* pRetryTmr; // used to dispatch data after a given time duration + void* pRspTmr; // used to dispatch data after a given time duration } SDispatchMsgInfo; typedef struct STaskQueue { @@ -356,8 +354,12 @@ typedef struct STaskExecStatisInfo { double step2El; int32_t updateCount; int64_t latestUpdateTs; - int32_t processDataBlocks; - int64_t processDataSize; + int32_t inputDataBlocks; + int64_t inputDataSize; + double procsThroughput; + int64_t outputDataBlocks; + int64_t outputDataSize; + double outputThroughput; int32_t dispatch; int64_t dispatchDataSize; int32_t checkpoint; @@ -431,7 +433,6 @@ struct SStreamTask { SHistoryTaskInfo hTaskInfo; STaskId streamTaskId; STaskExecStatisInfo execInfo; - SArray* pReadyMsgList; // SArray TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ SMsgCb* pMsgCb; // msg handle SStreamState* pState; // state backend @@ -517,6 +518,9 @@ typedef struct STaskUpdateEntry { int32_t transId; } STaskUpdateEntry; +typedef int32_t (*__state_trans_user_fn)(SStreamTask*, void* param); +typedef int32_t (*__stream_task_expand_fn)(struct SStreamTask* pTask); + SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam, SArray* pTaskList, bool hasFillhistory, int8_t subtableWithoutMd5); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); @@ -562,6 +566,8 @@ typedef struct STaskCkptInfo { int64_t latestId; // saved checkpoint id int64_t latestVer; // saved checkpoint ver int64_t latestTime; // latest checkpoint time + int64_t latestSize; // latest checkpoint size + int8_t remoteBackup; // latest checkpoint backup done int64_t activeId; // current active checkpoint id int32_t activeTransId; // checkpoint trans id int8_t failed; // denote if the checkpoint is failed or not @@ -579,8 +585,12 @@ typedef struct STaskStatusEntry { int64_t inputQUnchangeCounter; double inputQUsed; // in MiB double inputRate; - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dst data size + double procsThroughput; // duration between one element put into input queue and being processed. + double procsTotal; // duration between one element put into input queue and being processed. + double outputThroughput; // the size of dispatched result blocks in bytes + double outputTotal; // the size of dispatched result blocks in bytes + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size int64_t startTime; int64_t startCheckpointId; int64_t startCheckpointVer; @@ -605,7 +615,8 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask); int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pMsg); int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code); -SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); +SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); +SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); void streamTaskInputFail(SStreamTask* pTask); @@ -658,7 +669,18 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); +// checkpoint related +int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId); +int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId); +int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId); +bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId); +void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal); +void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask); +void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId); +int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pInfo, int32_t code); + int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); +int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue); // common void streamTaskPause(SStreamTask* pTask); @@ -668,10 +690,12 @@ int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstre void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); int32_t streamTaskReleaseState(SStreamTask* pTask); int32_t streamTaskReloadState(SStreamTask* pTask); +void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId); void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); +int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key); bool streamTaskIsSinkTask(const SStreamTask* pTask); +void streamTaskSetRemoveBackendFiles(SStreamTask* pTask); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); @@ -718,9 +742,9 @@ void streamMetaResetStartInfo(STaskStartInfo* pMeta); SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); void streamMetaLoadAllTasks(SStreamMeta* pMeta); -int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn fn); int32_t streamMetaStopAllTasks(SStreamMeta* pMeta); -int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); +int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, __stream_task_expand_fn fn); bool streamMetaAllTasksReady(const SStreamMeta* pMeta); // timer @@ -728,7 +752,9 @@ tmr_h streamTimerGetInstance(); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); -int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask); +int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp); +int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t downstreamTaskId); +int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg); int32_t streamAlignTransferState(SStreamTask* pTask); @@ -736,6 +762,10 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask); int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int32_t setCode); +int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId, + SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); +int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq); +SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); // stream task state machine, and event handling SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 25566d5acb..7f779609eb 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -45,6 +45,7 @@ extern "C" { #define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) typedef enum { + TAOS_WAL_SKIP = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2, } EWalType; @@ -145,17 +146,18 @@ typedef struct SWalReader SWalReader; // todo hide this struct struct SWalReader { - SWal *pWal; - int64_t readerId; - TdFilePtr pLogFile; - TdFilePtr pIdxFile; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t skipToVersion; // skip data and jump to destination version, usually used by stream resume ignoring untreated data + SWal *pWal; + int64_t readerId; + TdFilePtr pLogFile; + TdFilePtr pIdxFile; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t skipToVersion; // skip data and jump to destination version, usually used by stream resume ignoring untreated + // data int64_t capacity; TdThreadMutex mutex; SWalFilterCond cond; - SWalCkHead *pHead; + SWalCkHead *pHead; }; // module initialization @@ -198,7 +200,7 @@ SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond, int64_t id); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); -void decryptBody(SWalCfg* cfg, SWalCkHead* pHead, int32_t plainBodyLen, const char* func); +void decryptBody(SWalCfg *cfg, SWalCkHead *pHead, int32_t plainBodyLen, const char *func); int32_t walReaderSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); int64_t walReaderGetCurrentVer(const SWalReader *pReader); @@ -206,7 +208,7 @@ int64_t walReaderGetValidFirstVer(const SWalReader *pReader); int64_t walReaderGetSkipToVersion(SWalReader *pReader); void walReaderSetSkipToVersion(SWalReader *pReader, int64_t ver); void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever); -void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal* pOffset); +void walReaderVerifyOffset(SWalReader *pWalReader, STqOffsetVal *pOffset); // only for tq usage int32_t walFetchHead(SWalReader *pRead, int64_t ver); diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 48b7b8c56f..97bb6f53ba 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -114,6 +114,12 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); void *atomic_fetch_xor_ptr(void *ptr, void *val); +#ifdef _MSC_VER +#define tmemory_barrier(order) MemoryBarrier() +#else +#define tmemory_barrier(order) __sync_synchronize() +#endif + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index dafdac9649..8f8434dfc1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -327,7 +327,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_DB_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x0396) // #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x039A) #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE TAOS_DEF_ERROR_CODE(0, 0x039B) -#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x039C) // mnode-node #define TSDB_CODE_MND_MNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03A0) @@ -612,6 +611,16 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821) #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822) #define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823) +#define TSDB_CODE_GRANT_BASIC_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0824) +#define TSDB_CODE_GRANT_STREAM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0825) +#define TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0826) +#define TSDB_CODE_GRANT_VIEW_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0827) +#define TSDB_CODE_GRANT_AUDIT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0828) +#define TSDB_CODE_GRANT_CSV_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0829) +#define TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082A) +#define TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082B) +#define TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082C) +#define TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x082D) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x @@ -901,6 +910,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_STREAM_EXEC_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x4102) #define TSDB_CODE_STREAM_INVALID_STATETRANS TAOS_DEF_ERROR_CODE(0, 0x4103) #define TSDB_CODE_STREAM_TASK_IVLD_STATUS TAOS_DEF_ERROR_CODE(0, 0x4104) +#define TSDB_CODE_STREAM_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x4105) // TDLite #define TSDB_CODE_TDLITE_IVLD_OPEN_FLAGS TAOS_DEF_ERROR_CODE(0, 0x5100) diff --git a/include/util/tbuffer.inc b/include/util/tbuffer.inc index f0137ee978..2c1405d6c5 100644 --- a/include/util/tbuffer.inc +++ b/include/util/tbuffer.inc @@ -177,7 +177,7 @@ static int32_t tBufferReaderInit(SBufferReader *reader, uint32_t offset, SBuffer } static FORCE_INLINE int32_t tBufferGet(SBufferReader *reader, uint32_t size, void *data) { - if (reader->offset < 0 || reader->offset + size > reader->buffer->size) { + if (reader->offset + size > reader->buffer->size) { return TSDB_CODE_OUT_OF_RANGE; } if (data) { diff --git a/include/util/tdef.h b/include/util/tdef.h index 6ded54dc00..905a50886a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -78,6 +78,14 @@ extern const int32_t TYPE_BYTES[21]; #define TSDB_DEFAULT_PASS "taosdata" #endif +#ifndef TD_PRODUCT_NAME +#ifdef TD_ENTERPRISE +#define TD_PRODUCT_NAME "TDengine Enterprise Edition" +#else +#define TD_PRODUCT_NAME "TDengine Community Edition" +#endif +#endif + #define TSDB_TRUE 1 #define TSDB_FALSE 0 #define TSDB_OK 0 @@ -365,7 +373,7 @@ typedef enum ELogicConditionType { #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second -#define TSDB_MIN_WAL_LEVEL 1 +#define TSDB_MIN_WAL_LEVEL 0 #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI diff --git a/include/util/tencode.h b/include/util/tencode.h index 596fa2b4d3..b8da040689 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -77,6 +77,7 @@ typedef struct { #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) +#define TD_CODER_REMAIN_CAPACITY(CODER) ((CODER)->size - (CODER)->pos) #define tEncodeSize(E, S, SIZE, RET) \ do { \ diff --git a/include/util/tutil.h b/include/util/tutil.h index 54ce6fc849..c049949590 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -119,7 +119,7 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, #define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ do { \ - if ((CODE)) { \ + if (TSDB_CODE_SUCCESS != (CODE)) { \ LINO = __LINE__; \ goto LABEL; \ } \ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 21d9729c48..2fbf4eeabb 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -44,6 +44,7 @@ enum { RES_TYPE__TMQ, RES_TYPE__TMQ_META, RES_TYPE__TMQ_METADATA, + RES_TYPE__TMQ_BATCH_META, }; #define SHOW_VARIABLES_RESULT_COLS 3 @@ -51,10 +52,11 @@ enum { #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) -#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) -#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) -#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) -#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA) +#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) +#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) +#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) +#define TD_RES_TMQ_METADATA(res) (*(int8_t*)res == RES_TYPE__TMQ_METADATA) +#define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)res == RES_TYPE__TMQ_BATCH_META) typedef struct SAppInstInfo SAppInstInfo; @@ -242,6 +244,11 @@ typedef struct { STaosxRsp rsp; } SMqTaosxRspObj; +typedef struct { + SMqRspObjCommon common; + SMqBatchMetaRsp rsp; +} SMqBatchMetaRspObj; + typedef struct SReqRelInfo { uint64_t userRefId; uint64_t prevRefId; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 0c13265ad2..8110bb1a7d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -89,24 +89,28 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); - if (pRequest->pQuery && pRequest->pQuery->pRoot) { - if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && - (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) { - tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 - "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", - duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, - pRequest->metric.planCostUs, pRequest->metric.execCostUs); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); - reqType = SLOW_LOG_TYPE_INSERT; - } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { - tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 - "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", - duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, - pRequest->metric.planCostUs, pRequest->metric.execCostUs); + if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) { + if (pRequest->pQuery && pRequest->pQuery->pRoot) { + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && + (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) { + tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 + "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", + duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, + pRequest->metric.planCostUs, pRequest->metric.execCostUs); + atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + reqType = SLOW_LOG_TYPE_INSERT; + } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { + tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 + "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", + duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, + pRequest->metric.planCostUs, pRequest->metric.execCostUs); - atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); - reqType = SLOW_LOG_TYPE_QUERY; - } + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + reqType = SLOW_LOG_TYPE_QUERY; + } + } + + nodesSimReleaseAllocator(pRequest->allocatorRefId); } if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->stmtType || QUERY_NODE_INSERT_STMT == pRequest->stmtType) { @@ -492,7 +496,10 @@ void doDestroyRequest(void *p) { } taosMemoryFree(pRequest->body.interParam); - qDestroyQuery(pRequest->pQuery); + if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) { + qDestroyQuery(pRequest->pQuery); + nodesSimReleaseAllocator(pRequest->allocatorRefId); + } nodesDestroyAllocator(pRequest->allocatorRefId); taosMemoryFreeClear(pRequest->effectiveUser); @@ -726,7 +733,7 @@ void taos_init_imp(void) { return; } - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true) != 0) { + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7d30a19140..1b6cb8fd22 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -92,6 +92,7 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat } if (!pRsp) { releaseTscObj(pReq->connKey.tscRid); + taosHashCancelIterate(hbMgr->activeInfo, pReq); break; } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ba7f65c52b..da5da044a7 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -301,7 +301,7 @@ void taos_close(TAOS *taos) { } int taos_errno(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return terrno; } @@ -313,7 +313,7 @@ int taos_errno(TAOS_RES *res) { } const char *taos_errstr(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return (const char *)tstrerror(terrno); } @@ -354,6 +354,10 @@ void taos_free_result(TAOS_RES *res) { SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; tDeleteMqMetaRsp(&pRspObj->metaRsp); taosMemoryFree(pRspObj); + } else if (TD_RES_TMQ_BATCH_META(res)) { + SMqBatchMetaRspObj *pBtRspObj = (SMqBatchMetaRspObj *)res; + tDeleteMqBatchMetaRsp(&pBtRspObj->rsp); + taosMemoryFree(pBtRspObj); } } @@ -371,7 +375,7 @@ void taos_kill_query(TAOS *taos) { } int taos_field_count(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -382,7 +386,7 @@ int taos_field_count(TAOS_RES *res) { int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); } TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res)) { + if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -437,7 +441,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pResultInfo->current += 1; return pResultInfo->row; } - } else if (TD_RES_TMQ_META(res)) { + } else if (TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } else { // assert to avoid un-initialization error @@ -548,7 +552,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } int *taos_fetch_lengths(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -557,7 +561,7 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -625,7 +629,7 @@ const char *taos_get_client_info() { return version; } // return int32_t int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -636,7 +640,7 @@ int taos_affected_rows(TAOS_RES *res) { // return int64_t int64_t taos_affected_rows64(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -646,7 +650,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return TSDB_TIME_PRECISION_MILLI; } @@ -686,7 +690,7 @@ int taos_select_db(TAOS *taos, const char *db) { } void taos_stop_query(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return; } @@ -694,7 +698,7 @@ void taos_stop_query(TAOS_RES *res) { } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return true; } SReqResultInfo *pResultInfo = tscGetCurResInfo(res); @@ -719,7 +723,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -761,7 +765,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { *numOfRows = 0; *pData = NULL; - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -797,7 +801,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { - if (res == NULL || TD_RES_TMQ_META(res)) { + if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index c5069d46aa..ce404092b0 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -26,9 +26,13 @@ #define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64 #define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId +#define TMQ_META_VERSION "1.0" + +static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen); + static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } -static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, +static cJSON* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t, SColCmprWrapper* pColCmprRow) { int8_t buildDefaultCompress = 0; if (pColCmprRow->nCols <= 0) { @@ -122,9 +126,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch } cJSON_AddItemToObject(json, "tags", tags); - string = cJSON_PrintUnformatted(json); - cJSON_Delete(json); - return string; + return json; } static int32_t setCompressOption(cJSON* json, uint32_t para) { @@ -151,7 +153,7 @@ static int32_t setCompressOption(cJSON* json, uint32_t para) { } return 0; } -static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { +static cJSON* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { SMAlterStbReq req = {0}; cJSON* json = NULL; char* string = NULL; @@ -199,6 +201,26 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { } break; } + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { + SFieldWithOptions* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(field->type); + cJSON_AddItemToObject(json, "colType", colType); + + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || + field->type == TSDB_DATA_TYPE_GEOMETRY) { + int32_t length = field->bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + setCompressOption(json, field->compress); + break; + } case TSDB_ALTER_TABLE_DROP_TAG: case TSDB_ALTER_TABLE_DROP_COLUMN: { TAOS_FIELD* field = taosArrayGet(req.pFields, 0); @@ -245,18 +267,16 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { default: break; } - string = cJSON_PrintUnformatted(json); end: - cJSON_Delete(json); tFreeSMAltertbReq(&req); - return string; + return json; } -static char* processCreateStb(SMqMetaRsp* metaRsp) { +static cJSON* processCreateStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + cJSON* pJson = NULL; uDebug("create stable data:%p", metaRsp); // decode and process req @@ -267,17 +287,17 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { if (tDecodeSVCreateStbReq(&coder, &req) < 0) { goto _err; } - string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr); + pJson = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE, &req.colCmpr); _err: - uDebug("create stable return, sql json:%s", string); + uDebug("create stable return, sql json:%s", cJSON_PrintUnformatted(pJson)); tDecoderClear(&coder); - return string; + return pJson; } -static char* processAlterStb(SMqMetaRsp* metaRsp) { +static cJSON* processAlterStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + cJSON* pJson = NULL; uDebug("alter stable data:%p", metaRsp); // decode and process req @@ -288,11 +308,11 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { if (tDecodeSVCreateStbReq(&coder, &req) < 0) { goto _err; } - string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); + pJson = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); _err: - uDebug("alter stable return, sql json:%s", string); + uDebug("alter stable return, sql json:%s", cJSON_PrintUnformatted(pJson)); tDecoderClear(&coder); - return string; + return pJson; } static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { @@ -385,7 +405,7 @@ end: taosArrayDestroy(pTagVals); } -static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) { +static cJSON* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) { char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { @@ -410,16 +430,14 @@ static char* buildCreateCTableJson(SVCreateTbReq* pCreateReq, int32_t nReqs) { cJSON_AddItemToArray(createList, create); } cJSON_AddItemToObject(json, "createList", createList); - string = cJSON_PrintUnformatted(json); - cJSON_Delete(json); - return string; + return json; } -static char* processCreateTable(SMqMetaRsp* metaRsp) { +static cJSON* processCreateTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVCreateTbBatchReq req = {0}; SVCreateTbReq* pCreateReq; - char* string = NULL; + cJSON* pJson = NULL; // decode uDebug("create table data:%p", metaRsp); void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); @@ -433,25 +451,18 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { if (req.nReqs > 0) { pCreateReq = req.pReqs; if (pCreateReq->type == TSDB_CHILD_TABLE) { - string = buildCreateCTableJson(req.pReqs, req.nReqs); + pJson = buildCreateCTableJson(req.pReqs, req.nReqs); } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { - string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, + pJson = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE, &pCreateReq->colCmpr); } } _exit: - uDebug("create table return, sql json:%s", string); - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - taosMemoryFreeClear(pCreateReq->comment); - taosMemoryFreeClear(pCreateReq->sql); - if (pCreateReq->type == TSDB_CHILD_TABLE) { - taosArrayDestroy(pCreateReq->ctb.tagName); - } - } + uDebug("create table return, sql json:%s", cJSON_PrintUnformatted(pJson)); + tDeleteSVCreateTbBatchReq(&req); tDecoderClear(&decoder); - return string; + return pJson; } static char* processAutoCreateTable(STaosxRsp* rsp) { @@ -480,7 +491,9 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { goto _exit; } } - string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); + cJSON* pJson = buildCreateCTableJson(pCreateReq, rsp->createTableNum); + string = cJSON_PrintUnformatted(pJson); + cJSON_Delete(pJson); _exit: uDebug("auto created table return, sql json:%s", string); for (int i = 0; i < rsp->createTableNum; i++) { @@ -495,7 +508,7 @@ _exit: return string; } -static char* processAlterTable(SMqMetaRsp* metaRsp) { +static cJSON* processAlterTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVAlterTbReq vAlterTbReq = {0}; char* string = NULL; @@ -546,6 +559,25 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } break; } + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { + cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); + cJSON_AddItemToObject(json, "colName", colName); + cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); + cJSON_AddItemToObject(json, "colType", colType); + + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || + vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { + int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); + cJSON_AddItemToObject(json, "colLength", cbytes); + } + setCompressOption(json, vAlterTbReq.compress); + break; + } case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); @@ -620,19 +652,16 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { default: break; } - string = cJSON_PrintUnformatted(json); _exit: - uDebug("alter table return, sql json:%s", string); - cJSON_Delete(json); + uDebug("alter table return, sql json:%s", cJSON_PrintUnformatted(json)); tDecoderClear(&decoder); - return string; + return json; } -static char* processDropSTable(SMqMetaRsp* metaRsp) { +static cJSON* processDropSTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropStbReq req = {0}; - char* string = NULL; cJSON* json = NULL; uDebug("processDropSTable data:%p", metaRsp); @@ -657,18 +686,15 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { cJSON* tableName = cJSON_CreateString(req.name); cJSON_AddItemToObject(json, "tableName", tableName); - string = cJSON_PrintUnformatted(json); _exit: - uDebug("processDropSTable return, sql json:%s", string); - cJSON_Delete(json); + uDebug("processDropSTable return, sql json:%s", cJSON_PrintUnformatted(json)); tDecoderClear(&decoder); - return string; + return json; } -static char* processDeleteTable(SMqMetaRsp* metaRsp) { +static cJSON* processDeleteTable(SMqMetaRsp* metaRsp) { SDeleteRes req = {0}; SDecoder coder = {0}; cJSON* json = NULL; - char* string = NULL; uDebug("processDeleteTable data:%p", metaRsp); // decode and process req @@ -696,18 +722,15 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { cJSON* sqlJson = cJSON_CreateString(sql); cJSON_AddItemToObject(json, "sql", sqlJson); - string = cJSON_PrintUnformatted(json); _exit: - uDebug("processDeleteTable return, sql json:%s", string); - cJSON_Delete(json); + uDebug("processDeleteTable return, sql json:%s", cJSON_PrintUnformatted(json)); tDecoderClear(&coder); - return string; + return json; } -static char* processDropTable(SMqMetaRsp* metaRsp) { +static cJSON* processDropTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVDropTbBatchReq req = {0}; - char* string = NULL; cJSON* json = NULL; uDebug("processDropTable data:%p", metaRsp); @@ -741,12 +764,10 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { } cJSON_AddItemToObject(json, "tableNameList", tableNameList); - string = cJSON_PrintUnformatted(json); _exit: - uDebug("processDropTable return, json sql:%s", string); - cJSON_Delete(json); + uDebug("processDropTable return, json sql:%s", cJSON_PrintUnformatted(json)); tDecoderClear(&decoder); - return string; + return json; } static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { @@ -1121,13 +1142,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { end: uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - taosMemoryFreeClear(pCreateReq->comment); - if (pCreateReq->type == TSDB_CHILD_TABLE) { - taosArrayDestroy(pCreateReq->ctb.tagName); - } - } + tDeleteSVCreateTbBatchReq(&req); taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); @@ -1803,6 +1818,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { char err[ERR_MSG_LEN] = {0}; code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); + taosMemoryFreeClear(pTableMeta); if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; @@ -1995,6 +2011,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) char err[ERR_MSG_LEN] = {0}; code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true, err, ERR_MSG_LEN); taosMemoryFree(fields); + taosMemoryFreeClear(pTableMeta); if (code != TSDB_CODE_SUCCESS) { SET_ERROR_MSG("table:%s, err:%s", tbName, err); goto end; @@ -2025,38 +2042,85 @@ end: return code; } +static cJSON* processSimpleMeta(SMqMetaRsp* pMetaRsp) { + if (pMetaRsp->resMsgType == TDMT_VND_CREATE_STB) { + return processCreateStb(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_STB) { + return processAlterStb(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_STB) { + return processDropSTable(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_CREATE_TABLE) { + return processCreateTable(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_ALTER_TABLE) { + return processAlterTable(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) { + return processDropTable(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_DROP_TABLE) { + return processDropTable(pMetaRsp); + } else if (pMetaRsp->resMsgType == TDMT_VND_DELETE) { + return processDeleteTable(pMetaRsp); + } + + return NULL; +} +static char* processBatchMetaToJson(SMqBatchMetaRsp* pMsgRsp) { + SDecoder coder; + SMqBatchMetaRsp rsp = {0}; + tDecoderInit(&coder, pMsgRsp->pMetaBuff, pMsgRsp->metaBuffLen); + if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) { + goto _end; + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddStringToObject(pJson, "tmq_meta_version", TMQ_META_VERSION); + cJSON* pMetaArr = cJSON_CreateArray(); + int32_t num = taosArrayGetSize(rsp.batchMetaReq); + for (int32_t i = 0; i < num; i++) { + int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i); + void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i); + SDecoder metaCoder = {0}; + SMqMetaRsp metaRsp = {0}; + tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead)); + if(tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0 ) { + goto _end; + } + cJSON* pItem = processSimpleMeta(&metaRsp); + tDeleteMqMetaRsp(&metaRsp); + cJSON_AddItemToArray(pMetaArr, pItem); + } + + cJSON_AddItemToObject(pJson, "metas", pMetaArr); + tDeleteMqBatchMetaRsp(&rsp); + char* fullStr = cJSON_PrintUnformatted(pJson); + cJSON_Delete(pJson); + return fullStr; + +_end: + cJSON_Delete(pJson); + tDeleteMqBatchMetaRsp(&rsp); + return NULL; +} + char* tmq_get_json_meta(TAOS_RES* res) { if (res == NULL) return NULL; uDebug("tmq_get_json_meta res:%p", res); - if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res)) { + if (!TD_RES_TMQ_META(res) && !TD_RES_TMQ_METADATA(res) && !TD_RES_TMQ_BATCH_META(res)) { return NULL; } if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj* pMetaDataRspObj = (SMqTaosxRspObj*)res; return processAutoCreateTable(&pMetaDataRspObj->rsp); + } else if (TD_RES_TMQ_BATCH_META(res)) { + SMqBatchMetaRspObj* pBatchMetaRspObj = (SMqBatchMetaRspObj*)res; + return processBatchMetaToJson(&pBatchMetaRspObj->rsp); } SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { - return processCreateStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { - return processAlterStb(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { - return processDropSTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { - return processCreateTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { - return processAlterTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { - return processDropTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { - return processDropTable(&pMetaRspObj->metaRsp); - } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DELETE) { - return processDeleteTable(&pMetaRspObj->metaRsp); - } - - return NULL; + cJSON* pJson = processSimpleMeta(&pMetaRspObj->metaRsp); + char* string = cJSON_PrintUnformatted(pJson); + cJSON_Delete(pJson); + return string; } void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } @@ -2147,6 +2211,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { } raw->raw_type = RES_TYPE__TMQ_METADATA; uDebug("tmq get raw type metadata:%p", raw); + } else if (TD_RES_TMQ_BATCH_META(res)) { + SMqBatchMetaRspObj* pBtMetaRspObj = (SMqBatchMetaRspObj*)res; + raw->raw = pBtMetaRspObj->rsp.pMetaBuff; + raw->raw_len = pBtMetaRspObj->rsp.metaBuffLen; + raw->raw_type = RES_TYPE__TMQ_BATCH_META; + uDebug("tmq get raw batch meta:%p", raw); } else { uError("tmq get raw error type:%d", *(int8_t*)res); terrno = TSDB_CODE_TMQ_INVALID_MSG; @@ -2163,32 +2233,74 @@ void tmq_free_raw(tmq_raw_data raw) { memset(terrMsg, 0, ERR_MSG_LEN); } +static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) { + if (type == TDMT_VND_CREATE_STB) { + return taosCreateStb(taos, buf, len); + } else if (type == TDMT_VND_ALTER_STB) { + return taosCreateStb(taos, buf, len); + } else if (type == TDMT_VND_DROP_STB) { + return taosDropStb(taos, buf, len); + } else if (type == TDMT_VND_CREATE_TABLE) { + return taosCreateTable(taos, buf, len); + } else if (type == TDMT_VND_ALTER_TABLE) { + return taosAlterTable(taos, buf, len); + } else if (type == TDMT_VND_DROP_TABLE) { + return taosDropTable(taos, buf, len); + } else if (type == TDMT_VND_DELETE) { + return taosDeleteData(taos, buf, len); + } else if (type == RES_TYPE__TMQ) { + return tmqWriteRawDataImpl(taos, buf, len); + } else if (type == RES_TYPE__TMQ_METADATA) { + return tmqWriteRawMetaDataImpl(taos, buf, len); + } else if (type == RES_TYPE__TMQ_BATCH_META) { + return tmqWriteBatchMetaDataImpl(taos, buf, len); + } + return TSDB_CODE_INVALID_PARA; +} + int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { if (!taos) { - goto end; + return TSDB_CODE_INVALID_PARA; } - if (raw.raw_type == TDMT_VND_CREATE_STB) { - return taosCreateStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_ALTER_STB) { - return taosCreateStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DROP_STB) { - return taosDropStb(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_CREATE_TABLE) { - return taosCreateTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_ALTER_TABLE) { - return taosAlterTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DROP_TABLE) { - return taosDropTable(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == TDMT_VND_DELETE) { - return taosDeleteData(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == RES_TYPE__TMQ) { - return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len); - } else if (raw.raw_type == RES_TYPE__TMQ_METADATA) { - return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len); - } - -end: - terrno = TSDB_CODE_INVALID_PARA; - return terrno; + return writeRawImpl(taos, raw.raw, raw.raw_len, raw.raw_type); +} + +static int32_t tmqWriteBatchMetaDataImpl(TAOS* taos, void* meta, int32_t metaLen) { + if (taos == NULL || meta == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + SMqBatchMetaRsp rsp = {0}; + SDecoder coder; + int32_t code = TSDB_CODE_SUCCESS; + + // decode and process req + tDecoderInit(&coder, meta, metaLen); + if (tDecodeMqBatchMetaRsp(&coder, &rsp) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + int32_t num = taosArrayGetSize(rsp.batchMetaReq); + for (int32_t i = 0; i < num; i++) { + int32_t len = *(int32_t*)taosArrayGet(rsp.batchMetaLen, i); + void* tmpBuf = taosArrayGetP(rsp.batchMetaReq, i); + SDecoder metaCoder = {0}; + SMqMetaRsp metaRsp = {0}; + tDecoderInit(&metaCoder, POINTER_SHIFT(tmpBuf, sizeof(SMqRspHead)), len - sizeof(SMqRspHead)); + if (tDecodeMqMetaRsp(&metaCoder, &metaRsp) < 0) { + code = TSDB_CODE_INVALID_PARA; + goto _end; + } + code = writeRawImpl(taos, metaRsp.metaRsp, metaRsp.metaRspLen, metaRsp.resMsgType); + tDeleteMqMetaRsp(&metaRsp); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + } + +_end: + tDeleteMqBatchMetaRsp(&rsp); + errno = code; + return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6225cf703c..6eb7abe0eb 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -939,7 +939,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions)); for (int32_t i = 0; i < pReq.numOfColumns; ++i) { SField *pField = taosArrayGet(pColumns, i); - SFieldWithOptions fieldWithOption; + SFieldWithOptions fieldWithOption = {0}; setFieldWithOptions(&fieldWithOption, pField); setDefaultOptionsForField(&fieldWithOption); taosArrayPush(pReq.pColumns, &fieldWithOption); diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c index f715f32556..bc0e560178 100644 --- a/source/client/src/clientSmlTelnet.c +++ b/source/client/src/clientSmlTelnet.c @@ -233,7 +233,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine SSmlKv kvTs = {0}; smlBuildTsKv(&kvTs, ts); - if (needConverTime) { + if (needConverTime && info->currSTableMeta != NULL) { kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 907a6ebe3c..f5f083e5d8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -71,6 +71,7 @@ struct tmq_conf_t { char* pass; tmq_commit_cb* commitCb; void* commitCbUserParam; + int8_t enableBatchMeta; }; struct tmq_t { @@ -87,6 +88,7 @@ struct tmq_t { uint64_t consumerId; tmq_commit_cb* commitCb; void* commitCbUserParam; + int8_t enableBatchMeta; // status SRWLatch lock; @@ -171,9 +173,10 @@ typedef struct { uint64_t reqId; SEpSet* pEpset; union { - SMqDataRsp dataRsp; - SMqMetaRsp metaRsp; - STaosxRsp taosxRsp; + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; + STaosxRsp taosxRsp; + SMqBatchMetaRsp batchMetaRsp; }; } SMqPollRspWrapper; @@ -268,6 +271,7 @@ tmq_conf_t* tmq_conf_new() { conf->autoCommit = true; conf->autoCommitInterval = DEFAULT_AUTO_COMMIT_INTERVAL; conf->resetOffset = TMQ_OFFSET__RESET_LATEST; + conf->enableBatchMeta = false; return conf; } @@ -397,17 +401,22 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } + if (strcasecmp(key, "msg.enable.batchmeta") == 0) { + conf->enableBatchMeta = (taosStr2int64(value) != 0) ? true : false; + return TMQ_CONF_OK; + } + return TMQ_CONF_UNKNOWN; } tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } int32_t tmq_list_append(tmq_list_t* list, const char* src) { - if (list == NULL) return -1; + if (list == NULL) return TSDB_CODE_INVALID_PARA; SArray* container = &list->container; - if (src == NULL || src[0] == 0) return -1; + if (src == NULL || src[0] == 0) return TSDB_CODE_INVALID_PARA; char* topic = taosStrdup(src); - if (taosArrayPush(container, &topic) == NULL) return -1; + if (taosArrayPush(container, &topic) == NULL) return TSDB_CODE_INVALID_PARA; return 0; } @@ -635,6 +644,11 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c pTopicName = pRspObj->common.topic; vgId = pRspObj->common.vgId; offsetVal = pRspObj->rsp.common.rspOffset; + } else if (TD_RES_TMQ_BATCH_META(pRes)) { + SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)pRes; + pTopicName = pBtRspObj->common.topic; + vgId = pBtRspObj->common.vgId; + offsetVal = pBtRspObj->rsp.rspOffset; } else { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; @@ -812,7 +826,7 @@ void tmqSendHbReq(void* param, void* tmrId) { offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.endOffset; - offRows->ever = pVg->offsetInfo.walVerEnd; + offRows->ever = pVg->offsetInfo.walVerEnd == -1 ? 0 : pVg->offsetInfo.walVerEnd; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, @@ -858,7 +872,10 @@ void tmqSendHbReq(void* param, void* tmrId) { SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + if (code != 0) { + tscError("tmqSendHbReq asyncSendMsgToServer failed"); + } OVER: tDestroySMqHbReq(&req); @@ -934,7 +951,11 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); tDeleteSTaosxRsp(&pRsp->taosxRsp); - } + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFreeClear(pRsp->pEpset); + tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); + } } void tmqClearUnhandleMsg(tmq_t* tmq) { @@ -1112,6 +1133,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->resetOffsetCfg = conf->resetOffset; pTmq->replayEnable = conf->replayEnable; pTmq->sourceExcluded = conf->sourceExcluded; + pTmq->enableBatchMeta = conf->enableBatchMeta; if (conf->replayEnable) { pTmq->autoCommit = false; } @@ -1189,6 +1211,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.autoCommitInterval = tmq->autoCommitInterval; req.resetOffsetCfg = tmq->resetOffsetCfg; req.enableReplay = tmq->replayEnable; + req.enableBatchMeta = tmq->enableBatchMeta; for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1220,12 +1243,15 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(buf); goto FAIL; } SMqSubscribeCbParam param = {.rspErr = 0}; if (tsem_init(¶m.rspSem, 0, 0) != 0) { code = TSDB_CODE_TSC_INTERNAL_ERROR; + taosMemoryFree(buf); + taosMemoryFree(sendInfo); goto FAIL; } @@ -1245,10 +1271,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - // avoid double free if msg is sent - buf = NULL; - sendInfo = NULL; - tsem_wait(¶m.rspSem); tsem_destroy(¶m.rspSem); @@ -1260,7 +1282,10 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t retryCnt = 0; while ((code = syncAskEp(tmq)) != 0) { if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes", tmq->consumerId); + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, strerror(code)); + if(code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { + code = 0; + } goto FAIL; } @@ -1284,8 +1309,6 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { FAIL: taosArrayDestroyP(req.topicNames, taosMemoryFree); - taosMemoryFree(buf); - taosMemoryFree(sendInfo); return code; } @@ -1358,7 +1381,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if(pMsg->pData == NULL){ tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); - goto FAIL; + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto END; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; @@ -1387,7 +1411,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; + goto END; } tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); @@ -1403,7 +1427,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; + goto END; } tDecoderClear(&decoder); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); @@ -1414,10 +1438,23 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; + goto END; } tDecoderClear(&decoder); memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); + } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + SDecoder decoder; + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); + if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){ + tDecoderClear(&decoder); + taosReleaseRef(tmqMgmt.rsetId, refId); + code = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + tDecoderClear(&decoder); + memcpy(&pRspWrapper->batchMetaRsp, pMsg->pData, sizeof(SMqRspHead)); + tscDebug("consumer:0x%" PRIx64 " recv poll batchmeta rsp, vgId:%d, reqId:0x%" PRIx64, tmq->consumerId, vgId, + requestId); } else { // invalid rspType tscError("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); } @@ -1600,6 +1637,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->reqId = generateRequestId(); pReq->enableReplay = tmq->replayEnable; pReq->sourceExcluded = tmq->sourceExcluded; + pReq->enableBatchMeta = tmq->enableBatchMeta; } SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { @@ -1616,6 +1654,20 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { return pRspObj; } +SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { + SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); + if(pRspObj == NULL) { + return NULL; + } + pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; + tstrncpy(pRspObj->common.topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->common.db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->common.vgId = pWrapper->vgHandle->vgId; + + memcpy(&pRspObj->rsp, &pWrapper->batchMetaRsp, sizeof(SMqBatchMetaRsp)); + tscDebug("build batchmeta Rsp from wrapper"); + return pRspObj; +} void changeByteEndian(char* pData){ char* p = pData; @@ -1744,6 +1796,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(msg); goto FAIL; } @@ -1755,6 +1808,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(msg); goto FAIL; } @@ -1781,7 +1835,6 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p return 0; FAIL: - taosMemoryFreeClear(msg); return tmqPollCb(pParam, NULL, code); } @@ -1805,14 +1858,15 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 10ms + int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; + if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } - if (tmq->replayEnable && - taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms + elapsed = taosGetTimestampMs() - pVg->blockReceiveTs; + if (tmq->replayEnable && elapsed < pVg->blockSleepForReplay && elapsed >= 0) { tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); continue; @@ -1886,8 +1940,6 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { terrno = pRspWrapper->code; tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(pRspWrapper->code)); - taosFreeQitem(pRspWrapper); - return NULL; } else { if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform askEp(tmq, NULL, false, true); @@ -2005,6 +2057,42 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); } + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + + tscDebug("consumer:0x%" PRIx64 " process meta rsp", tmq->consumerId); + + if (pollRspWrapper->batchMetaRsp.head.epoch == consumerEpoch) { + taosWLockLatch(&tmq->lock); + SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + pollRspWrapper->vgHandle = pVg; + pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); + if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { + tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, + pollRspWrapper->topicName, pollRspWrapper->vgId); + tmqFreeRspWrapper(pRspWrapper); + taosFreeQitem(pRspWrapper); + taosWUnLockLatch(&tmq->lock); + return NULL; + } + + // build rsp + void* pRsp = NULL; + updateVgInfo(pVg, &pollRspWrapper->batchMetaRsp.rspOffset, &pollRspWrapper->batchMetaRsp.rspOffset, + pollRspWrapper->batchMetaRsp.head.walsver, pollRspWrapper->batchMetaRsp.head.walever, + tmq->consumerId, true); + pRsp = tmqBuildBatchMetaRspFromWrapper(pollRspWrapper); + taosFreeQitem(pRspWrapper); + taosWUnLockLatch(&tmq->lock); + return pRsp; + } else { + tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->batchMetaRsp.head.epoch, consumerEpoch); + setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); + tmqFreeRspWrapper(pRspWrapper); + taosFreeQitem(pRspWrapper); + } } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); @@ -2127,7 +2215,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { if (timeout >= 0) { int64_t currentTime = taosGetTimestampMs(); int64_t elapsedTime = currentTime - startTime; - if (elapsedTime > timeout) { + if (elapsedTime > timeout || elapsedTime < 0) { tscDebug("consumer:0x%" PRIx64 " (epoch %d) timeout, no rsp, start time %" PRId64 ", current time %" PRId64, tmq->consumerId, tmq->epoch, startTime, currentTime); return NULL; @@ -2216,6 +2304,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { return TMQ_RES_TABLE_META; } else if (TD_RES_TMQ_METADATA(res)) { return TMQ_RES_METADATA; + } else if (TD_RES_TMQ_BATCH_META(res)) { + return TMQ_RES_TABLE_META; } else { return TMQ_RES_INVALID; } @@ -2225,7 +2315,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return strchr(((SMqRspObjCommon*)res)->topic, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; @@ -2240,7 +2330,7 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return strchr(((SMqRspObjCommon*)res)->db, '.') + 1; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; @@ -2252,15 +2342,15 @@ const char* tmq_get_db_name(TAOS_RES* res) { int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { - return -1; + return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return ((SMqRspObjCommon*)res)->vgId; } else if (TD_RES_TMQ_META(res)) { SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; return pMetaRspObj->vgId; } else { - return -1; + return TSDB_CODE_INVALID_PARA; } } @@ -2281,6 +2371,11 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { return pRspObj->metaRsp.rspOffset.version; } + } else if (TD_RES_TMQ_BATCH_META(res)) { + SMqBatchMetaRspObj* pBtRspObj = (SMqBatchMetaRspObj*)res; + if (pBtRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { + return pBtRspObj->rsp.rspOffset.version; + } } else { tscError("invalid tmq type:%d", *(int8_t*)res); } @@ -2558,6 +2653,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", pTmq->consumerId, tlen); code = TSDB_CODE_INVALID_PARA; + taosMemoryFree(pReq); goto FAIL; } @@ -2565,6 +2661,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { if (pParam == NULL) { tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", pTmq->consumerId); code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pReq); goto FAIL; } @@ -2575,6 +2672,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pReq); goto FAIL; } @@ -2596,8 +2694,6 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { } FAIL: - taosMemoryFreeClear(pParam); - taosMemoryFreeClear(pReq); askEpCb(pParam, NULL, code); } @@ -2612,7 +2708,7 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { if (tmq == NULL) { taosMemoryFree(pParamSet); terrno = TSDB_CODE_TMQ_CONSUMER_CLOSED; - return -1; + return terrno; } // if no more waiting rsp @@ -2704,7 +2800,6 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); - taosMemoryFree(pParam); return 0; } @@ -2808,8 +2903,6 @@ int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* ep int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); if (code != 0) { - taosMemoryFree(buf); - taosMemoryFree(sendInfo); tsem_destroy(&pParam->sem); taosMemoryFree(pParam); return code; @@ -3066,6 +3159,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; + sendInfo->paramFreeFp = taosMemoryFree; sendInfo->fp = tmqGetWalInfoCb; sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; @@ -3077,8 +3171,6 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); if (code != 0) { - taosMemoryFree(pParam); - taosMemoryFree(msg); goto end; } } @@ -3234,8 +3326,6 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); if (code != 0) { - taosMemoryFree(msg); - taosMemoryFree(sendInfo); tsem_destroy(&pParam->sem); taosMemoryFree(pParam); return code; diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index e448aec5e0..2ed21616dc 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -114,29 +114,34 @@ static int32_t execCommand(char* command){ void stopRsync() { int32_t code = #ifdef WINDOWS - system("taskkill /f /im rsync.exe"); + system("taskkill /f /im rsync.exe"); #else - system("pkill rsync"); + system("pkill rsync"); #endif - if(code != 0){ - uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); - return; + + if (code != 0) { + uError("[rsync] stop rsync server failed," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + } else { + uDebug("[rsync] stop rsync server successful"); } - uDebug("[rsync] stop rsync server successful"); + + taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. } void startRsync() { - if(taosMulMkDir(tsCheckpointBackupDir) != 0){ - uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA); + if (taosMulMkDir(tsCheckpointBackupDir) != 0) { + uError("[rsync] build checkpoint backup dir failed, path:%s," ERRNO_ERR_FORMAT, tsCheckpointBackupDir, + ERRNO_ERR_DATA); return; } + removeEmptyDir(); char confDir[PATH_MAX] = {0}; snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir); int32_t code = generateConfigFile(confDir); - if(code != 0){ + if (code != 0) { return; } @@ -144,25 +149,25 @@ void startRsync() { snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir); // start rsync service to backup checkpoint code = system(cmd); - if(code != 0){ - uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return; + if (code != 0) { + uError("[rsync] start server failed, code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + } else { + uDebug("[rsync] start server successful"); } - uDebug("[rsync] start server successful"); } -int32_t uploadRsync(const char* id, const char* path) { +int32_t uploadByRsync(const char* id, const char* path) { + int64_t st = taosGetTimestampMs(); + char command[PATH_MAX] = {0}; + #ifdef WINDOWS char pathTransform[PATH_MAX] = {0}; changeDirFromWindowsToLinux(path, pathTransform); -#endif - char command[PATH_MAX] = {0}; -#ifdef WINDOWS - if(pathTransform[strlen(pathTransform) - 1] != '/'){ + if(pathTransform[strlen(pathTransform) - 1] != '/') { #else - if(path[strlen(path) - 1] != '/'){ + if (path[strlen(path) - 1] != '/') { #endif snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", #ifdef WINDOWS @@ -178,26 +183,37 @@ int32_t uploadRsync(const char* id, const char* path) { #else path #endif - , tsSnodeAddress, id); + , + tsSnodeAddress, id); } int32_t code = execCommand(command); - if(code != 0){ - uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return -1; + if (code != 0) { + uError("[rsync] s-task:%s upload checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, + tsSnodeAddress, code, ERRNO_ERR_DATA); + } else { + int64_t el = (taosGetTimestampMs() - st); + uDebug("[rsync] s-task:%s upload checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, + tsSnodeAddress, el); } - uDebug("[rsync] upload data:%s successful", id); - return 0; + return code; } +// abort from retry if quit int32_t downloadRsync(const char* id, const char* path) { + int64_t st = taosGetTimestampMs(); + int32_t MAX_RETRY = 60; + int32_t times = 0; + int32_t code = 0; + #ifdef WINDOWS char pathTransform[PATH_MAX] = {0}; changeDirFromWindowsToLinux(path, pathTransform); #endif + char command[PATH_MAX] = {0}; - snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", + snprintf(command, PATH_MAX, "rsync -av --debug=all --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", tsSnodeAddress, id, #ifdef WINDOWS pathTransform @@ -206,14 +222,22 @@ int32_t downloadRsync(const char* id, const char* path) { #endif ); - int32_t code = execCommand(command); - if (code != 0) { - uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return -1; + uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command); + + while(times++ < MAX_RETRY) { + code = execCommand(command); + if (code != TSDB_CODE_SUCCESS) { + uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id, + path, times, code, ERRNO_ERR_DATA); + taosSsleep(1); + } else { + int32_t el = taosGetTimestampMs() - st; + uDebug("[rsync] %s download checkpoint data:%s successfully, elapsed time:%dms", id, path, el); + break; + } } - uDebug("[rsync] down data:%s successful", id); - return 0; + return code; } int32_t deleteRsync(const char* id) { diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 9de682dd3a..93167a1ccd 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -76,6 +76,7 @@ static const SSysDbTableSchema arbGroupsSchema[] = { {.name = "is_sync", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "assigned_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "assigned_token", .bytes = TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "assigned_acked", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, }; static const SSysDbTableSchema clusterSchema[] = { @@ -187,16 +188,23 @@ static const SSysDbTableSchema streamTaskSchema[] = { {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "process_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "process_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "out_total", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "out_throughput", .bytes = 14, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, +// {.name = "dispatch_throughput", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, +// {.name = "dispatch_total", .bytes = 12, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "info", .bytes = 35, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "info", .bytes = 40+ VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "start_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "checkpoint_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, - {.name = "checkpoint_version", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, - {.name = "checkpoint_backup", .bytes = 15, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "ds_err_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "checkpoint_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, + {.name = "checkpoint_size", .bytes = 14 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "checkpoint_backup", .bytes = 14 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "extra_info", .bytes = 25 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index fc9dd1fb4c..ba36558587 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -131,6 +131,7 @@ const char* columnEncodeStr(uint8_t type) { encode = TSDB_COLUMN_ENCODE_DISABLED; break; default: + encode = TSDB_COLUMN_ENCODE_UNKNOWN; break; } return encode; @@ -157,8 +158,8 @@ const char* columnCompressStr(uint16_t type) { case TSDB_COLVAL_COMPRESS_DISABLED: compress = TSDB_COLUMN_COMPRESS_DISABLED; break; - default: + compress = TSDB_COLUMN_COMPRESS_UNKNOWN; break; } return compress; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1c0b696aaa..ac4811fb1b 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2435,18 +2435,13 @@ void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId // the total length is fixed to be 34 bytes. bool isAutoTableName(char* ctbName) { return (strlen(ctbName) == 34 && ctbName[0] == 't' && ctbName[1] == '_'); } -bool alreadyAddGroupId(char* ctbName) { - size_t len = strlen(ctbName); - if (len == 0) return false; - size_t _location = len - 1; - while (_location > 0) { - if (ctbName[_location] < '0' || ctbName[_location] > '9') { - break; - } - _location--; - } - - return ctbName[_location] == '_' && len - 1 - _location >= 15; // 15 means the min length of groupid +bool alreadyAddGroupId(char* ctbName, int64_t groupId) { + char tmp[64] = {0}; + snprintf(tmp, sizeof(tmp), "%" PRIu64, groupId); + size_t len1 = strlen(ctbName); + size_t len2 = strlen(tmp); + if (len1 < len2) return false; + return memcmp(ctbName + len1 - len2, tmp, len2) == 0; } char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ae3fe6a2a0..0e80d431d1 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -383,7 +383,9 @@ static int32_t tRowBuildKVRow(SArray *aColVal, const SRowBuildScanInfo *sinfo, c if (IS_VAR_DATA_TYPE(schema->columns[i].type)) { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); payloadSize += tPutU32v(payload + payloadSize, colValArray[colValIndex].value.nData); - memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + if (colValArray[colValIndex].value.nData > 0) { + memcpy(payload + payloadSize, colValArray[colValIndex].value.pData, colValArray[colValIndex].value.nData); + } payloadSize += colValArray[colValIndex].value.nData; } else { payloadSize += tPutI16v(payload + payloadSize, colValArray[colValIndex].cid); @@ -1316,6 +1318,7 @@ int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc) { pVal->val = pSrc->pks[i].val; } else { pVal->nData = pSrc->pks[i].nData; + ASSERT(pSrc->pks[i].pData != NULL); memcpy(pVal->pData, pSrc->pks[i].pData, pVal->nData); } } @@ -2907,7 +2910,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt } } else { if (varDataTLen(data + offset) > bytes) { - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset), bytes); code = TSDB_CODE_PAR_VALUE_TOO_LONG; goto _exit; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 650a639964..f034244c69 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -273,7 +273,8 @@ int32_t tsCompactPullupInterval = 10; int32_t tsMqRebalanceInterval = 2; int32_t tsStreamCheckpointInterval = 60; float tsSinkDataRate = 2.0; -int32_t tsStreamNodeCheckInterval = 16; +int32_t tsStreamNodeCheckInterval = 20; +int32_t tsMaxConcurrentCheckpoint = 1; int32_t tsTtlUnit = 86400; int32_t tsTtlPushIntervalSec = 10; int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups @@ -621,114 +622,80 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - - tsNumOfSupportVnodes = tsNumOfCores * 2; - tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; - - if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - // if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - - if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) - return -1; - if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) - return -1; - - if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + + tsNumOfSupportVnodes = tsNumOfCores * 2 + 5; + tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); - if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; tsNumOfVnodeQueryThreads = tsNumOfCores * 2; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 16); - if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != - 0) - return -1; - - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != - 0) - return -1; tsNumOfVnodeRsmaThreads = tsNumOfCores / 4; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 16); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != - 0) - return -1; + + tsNumOfSnodeStreamThreads = tsNumOfCores / 4; + tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); + + tsNumOfSnodeWriteThreads = tsNumOfCores / 4; + tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); + + tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; + tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); + + // clang-format off + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + + if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); // if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; - tsNumOfSnodeStreamThreads = tsNumOfCores / 4; - tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - tsNumOfSnodeWriteThreads = tsNumOfCores / 4; - tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != - 0) - return -1; + if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; - tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); - if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, - CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != - 0) - return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), - CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "arbCheckSyncIntervalSec", tsArbCheckSyncIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "arbSetAssignedTimeoutSec", tsArbSetAssignedTimeoutSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; - - if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -736,9 +703,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, - CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -752,68 +717,43 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; if (cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) - return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; - if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; + if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; - if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; - if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, - CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; + if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "concurrentCheckpoint", tsMaxConcurrentCheckpoint, 1, 10, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -823,41 +763,23 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; - if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) - return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - /* - if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; - if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) { - uError("failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", tsS3BlockSize); - return -1; - } - if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != - 0) - return -1; - */ - if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, - CFG_DYN_ENT_SERVER) != 0) - return -1; + + if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; // min free disk space used to check if the disk is full [50MB, 1GB] - if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, - CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) - return -1; + if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + // clang-format on + // GRANT_CFG_ADD; return 0; } @@ -878,7 +800,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem = cfgGetItem(tsCfg, "supportVnodes"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsNumOfSupportVnodes = numOfCores * 2; + tsNumOfSupportVnodes = numOfCores * 2 + 5; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); pItem->i32 = tsNumOfSupportVnodes; pItem->stype = stype; @@ -1377,6 +1299,35 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } +int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs) { + if (tsCfg == NULL) osDefaultInit(); + + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; + + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + + if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { + printf("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (cfgLoadFromArray(pCfg, pArgs) != 0) { + printf("failed to load cfg from array since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + + cfgCleanup(pCfg); + return 0; +} + static int32_t taosCheckGlobalCfg() { uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn); if (ipv4 == 0xffffffff) { @@ -1394,7 +1345,7 @@ static int32_t taosCheckGlobalCfg() { } int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, - bool tsc, bool isDumpCfg) { + bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -1441,7 +1392,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32); - if(isDumpCfg) cfgDumpCfg(tsCfg, tsc, false); + cfgDumpCfg(tsCfg, tsc, false); if (taosCheckGlobalCfg() != 0) { return -1; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 778f149a58..1145da248f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -768,11 +768,22 @@ int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1; + + // if (pReq->alterType == ) for (int32_t i = 0; i < pReq->numOfFields; ++i) { - SField *pField = taosArrayGet(pReq->pFields, i); - if (tEncodeI8(&encoder, pField->type) < 0) return -1; - if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; - if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) { + SFieldWithOptions *pField = taosArrayGet(pReq->pFields, i); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + if (tEncodeU32(&encoder, pField->compress) < 0) return -1; + + } else { + SField *pField = taosArrayGet(pReq->pFields, i); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; + } } if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; @@ -802,13 +813,28 @@ int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq } for (int32_t i = 0; i < pReq->numOfFields; ++i) { - SField field = {0}; - if (tDecodeI8(&decoder, &field.type) < 0) return -1; - if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; - if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; - if (taosArrayPush(pReq->pFields, &field) == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + if (pReq->alterType == TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION) { + + taosArrayDestroy(pReq->pFields); + pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SFieldWithOptions)); + SFieldWithOptions field = {0}; + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (tDecodeU32(&decoder, &field.compress) < 0) return -1; + if (taosArrayPush(pReq->pFields, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } else { + SField field = {0}; + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; + if (taosArrayPush(pReq->pFields, &field) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } } } @@ -5616,18 +5642,36 @@ int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq * } int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) { + int32_t code = 0; SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); - if (tStartDecode(&decoder) < 0) return -1; + if (tStartDecode(&decoder) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + goto _exit; + } - if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->compactId) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + goto _exit; + } + + if (tDecodeI32(&decoder, &pReq->vgId) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + goto _exit; + } + + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + goto _exit; + } tEndDecode(&decoder); + +_exit: tDecoderClear(&decoder); - return 0; + return code; } int32_t tSerializeSAlterVnodeConfigReq(void *buf, int32_t bufLen, SAlterVnodeConfigReq *pReq) { @@ -6437,21 +6481,33 @@ void tFreeSVArbSetAssignedLeaderRsp(SVArbSetAssignedLeaderRsp *pRsp) { taosMemoryFreeClear(pRsp->memberToken); } -int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) { +int32_t tSerializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; - if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - if (tEncodeI32(&encoder, pReq->members[i].dnodeId) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->members[i].token) < 0) return -1; + + int32_t sz = taosArrayGetSize(pReq->updateArray); + if (tEncodeI32(&encoder, sz) < 0) return -1; + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + if (tEncodeI32(&encoder, pGroup->vgId) < 0) return -1; + if (tEncodeI64(&encoder, pGroup->dbUid) < 0) return -1; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + if (tEncodeI32(&encoder, pGroup->members[i].dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pGroup->members[i].token) < 0) return -1; + } + if (tEncodeI8(&encoder, pGroup->isSync) < 0) return -1; + if (tEncodeI32(&encoder, pGroup->assignedLeader.dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pGroup->assignedLeader.token) < 0) return -1; + if (tEncodeI64(&encoder, pGroup->version) < 0) return -1; + } + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + if (tEncodeI8(&encoder, pGroup->assignedLeader.acked) < 0) return -1; } - if (tEncodeI8(&encoder, pReq->isSync) < 0) return -1; - if (tEncodeI32(&encoder, pReq->assignedLeader.dnodeId) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->assignedLeader.token) < 0) return -1; - if (tEncodeI64(&encoder, pReq->version) < 0) return -1; tEndEncode(&encoder); @@ -6460,23 +6516,44 @@ int32_t tSerializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGrou return tlen; } -int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGroupReq *pReq) { +int32_t tDeserializeSMArbUpdateGroupBatchReq(void *buf, int32_t bufLen, SMArbUpdateGroupBatchReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - if (tDecodeI32(&decoder, &pReq->members[i].dnodeId) < 0) return -1; - pReq->members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); - if (tDecodeCStrTo(&decoder, pReq->members[i].token) < 0) return -1; + int32_t sz = 0; + if (tDecodeI32(&decoder, &sz) < 0) return -1; + + SArray *updateArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup)); + if (!updateArray) return -1; + + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup group = {0}; + if (tDecodeI32(&decoder, &group.vgId) < 0) return -1; + if (tDecodeI64(&decoder, &group.dbUid) < 0) return -1; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + if (tDecodeI32(&decoder, &group.members[i].dnodeId) < 0) return -1; + group.members[i].token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); + if (tDecodeCStrTo(&decoder, group.members[i].token) < 0) return -1; + } + if (tDecodeI8(&decoder, &group.isSync) < 0) return -1; + if (tDecodeI32(&decoder, &group.assignedLeader.dnodeId) < 0) return -1; + group.assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); + if (tDecodeCStrTo(&decoder, group.assignedLeader.token) < 0) return -1; + if (tDecodeI64(&decoder, &group.version) < 0) return -1; + group.assignedLeader.acked = false; + + taosArrayPush(updateArray, &group); } - if (tDecodeI8(&decoder, &pReq->isSync) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->assignedLeader.dnodeId) < 0) return -1; - pReq->assignedLeader.token = taosMemoryMalloc(TSDB_ARB_TOKEN_SIZE); - if (tDecodeCStrTo(&decoder, pReq->assignedLeader.token) < 0) return -1; - if (tDecodeI64(&decoder, &pReq->version) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(updateArray, i); + if (tDecodeI8(&decoder, &pGroup->assignedLeader.acked) < 0) return -1; + } + } + + pReq->updateArray = updateArray; tEndDecode(&decoder); @@ -6484,14 +6561,20 @@ int32_t tDeserializeSMArbUpdateGroupReq(void *buf, int32_t bufLen, SMArbUpdateGr return 0; } -void tFreeSMArbUpdateGroupReq(SMArbUpdateGroupReq *pReq) { - if (NULL == pReq) { +void tFreeSMArbUpdateGroupBatchReq(SMArbUpdateGroupBatchReq *pReq) { + if (NULL == pReq || NULL == pReq->updateArray) { return; } - for (int i = 0; i < 2; i++) { - taosMemoryFreeClear(pReq->members[i].token); + + int32_t sz = taosArrayGetSize(pReq->updateArray); + for (int32_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pGroup = taosArrayGet(pReq->updateArray, i); + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + taosMemoryFreeClear(pGroup->members[i].token); + } + taosMemoryFreeClear(pGroup->assignedLeader.token); } - taosMemoryFreeClear(pReq->assignedLeader.token); + taosArrayDestroy(pReq->updateArray); } // int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { @@ -7201,6 +7284,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { pHead->vgId = htonl(pReq->head.vgId); pHead->contLen = htonl(tlen + headLen); } + if (tEncodeI8(&encoder, pReq->enableBatchMeta) < 0) return -1; return tlen + headLen; } @@ -7230,6 +7314,12 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { if (tDecodeI8(&decoder, &pReq->sourceExcluded) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->enableBatchMeta) < 0) return -1; + } else { + pReq->enableBatchMeta = false; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -8364,6 +8454,18 @@ int tDecodeSVCreateTbBatchReq(SDecoder *pCoder, SVCreateTbBatchReq *pReq) { return 0; } +void tDeleteSVCreateTbBatchReq(SVCreateTbBatchReq *pReq) { + for (int32_t iReq = 0; iReq < pReq->nReqs; iReq++) { + SVCreateTbReq *pCreateReq = pReq->pReqs + iReq; + taosMemoryFreeClear(pCreateReq->sql); + taosMemoryFreeClear(pCreateReq->comment); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + taosArrayDestroy(pCreateReq->ctb.tagName); + pCreateReq->ctb.tagName = NULL; + } + } +} + int tEncodeSVCreateTbRsp(SEncoder *pCoder, const SVCreateTbRsp *pRsp) { if (tStartEncode(pCoder) < 0) return -1; @@ -8718,6 +8820,13 @@ int32_t tEncodeSVAlterTbReq(SEncoder *pEncoder, const SVAlterTbReq *pReq) { if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1; break; + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + if (tEncodeCStr(pEncoder, pReq->colName) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->type) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pEncoder, pReq->bytes) < 0) return -1; + if (tEncodeU32(pEncoder, pReq->compress) < 0) return -1; + break; default: break; } @@ -8773,6 +8882,12 @@ static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq) if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1; break; + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + if (tDecodeCStr(pDecoder, &pReq->colName) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pDecoder, &pReq->bytes) < 0) return -1; + if (tDecodeU32(pDecoder, &pReq->compress) < 0) return -1; default: break; } @@ -10559,3 +10674,55 @@ void tFreeFetchTtlExpiredTbsRsp(void *p) { SVFetchTtlExpiredTbsRsp *pRsp = p; taosArrayDestroy(pRsp->pExpiredTbs); } + +int32_t tEncodeMqBatchMetaRsp(SEncoder *pEncoder, const SMqBatchMetaRsp *pRsp) { + if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; + + int32_t size = taosArrayGetSize(pRsp->batchMetaReq); + if (tEncodeI32(pEncoder, size) < 0) return -1; + if (size > 0) { + for (int32_t i = 0; i < size; i++) { + void *pMetaReq = taosArrayGetP(pRsp->batchMetaReq, i); + int32_t metaLen = *(int32_t *)taosArrayGet(pRsp->batchMetaLen, i); + if (tEncodeBinary(pEncoder, pMetaReq, metaLen) < 0) return -1; + } + } + return 0; +} + +int32_t tDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) { + int32_t size = 0; + if (tDecodeI32(pDecoder, &size) < 0) return -1; + if (size > 0) { + pRsp->batchMetaReq = taosArrayInit(size, POINTER_BYTES); + pRsp->batchMetaLen = taosArrayInit(size, sizeof(int32_t)); + for (int32_t i = 0; i < size; i++) { + void *pCreate = NULL; + uint64_t len = 0; + if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1; + int32_t l = (int32_t)len; + taosArrayPush(pRsp->batchMetaReq, &pCreate); + taosArrayPush(pRsp->batchMetaLen, &l); + } + } + return 0; +} + +int32_t tSemiDecodeMqBatchMetaRsp(SDecoder *pDecoder, SMqBatchMetaRsp *pRsp) { + if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; + if (pDecoder->size < pDecoder->pos) { + return -1; + } + pRsp->metaBuffLen = TD_CODER_REMAIN_CAPACITY(pDecoder); + pRsp->pMetaBuff = taosMemoryCalloc(1, pRsp->metaBuffLen); + memcpy(pRsp->pMetaBuff, TD_CODER_CURRENT(pDecoder), pRsp->metaBuffLen); + return 0; +} + +void tDeleteMqBatchMetaRsp(SMqBatchMetaRsp *pRsp) { + taosMemoryFreeClear(pRsp->pMetaBuff); + taosArrayDestroyP(pRsp->batchMetaReq, taosMemoryFree); + taosArrayDestroy(pRsp->batchMetaLen); + pRsp->batchMetaReq = NULL; + pRsp->batchMetaLen = NULL; +} diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 8e0e50165f..d30b26e564 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -692,4 +692,38 @@ TEST(timeTest, epSet) { ASSERT_EQ(ep.numOfEps, 1); } } + +// Define test cases +TEST(AlreadyAddGroupIdTest, GroupIdAdded) { + // Test case 1: Group ID has been added + char ctbName[64] = "abc123"; + int64_t groupId = 123; + bool result = alreadyAddGroupId(ctbName, groupId); + EXPECT_TRUE(result); +} + +TEST(AlreadyAddGroupIdTest, GroupIdNotAdded) { + // Test case 2: Group ID has not been added + char ctbName[64] = "abc456"; + int64_t groupId = 123; + bool result = alreadyAddGroupId(ctbName, groupId); + EXPECT_FALSE(result); +} + +TEST(AlreadyAddGroupIdTest, GroupIdAddedAtTheEnd) { + // Test case 3: Group ID has been added at the end + char ctbName[64] = "xyz1"; + int64_t groupId = 1; + bool result = alreadyAddGroupId(ctbName, groupId); + EXPECT_TRUE(result); +} + +TEST(AlreadyAddGroupIdTest, GroupIdAddedWithDifferentLength) { + // Test case 4: Group ID has been added with different length + char ctbName[64] = "def"; + int64_t groupId = 123456; + bool result = alreadyAddGroupId(ctbName, groupId); + EXPECT_FALSE(result); +} + #pragma GCC diagnostic pop diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index f7fcd7697f..786df6b907 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -205,11 +205,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { if(i < argc - 1) { int32_t len = strlen(argv[++i]); if (len < ENCRYPT_KEY_LEN_MIN) { - printf("Error: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN); + printf("ERROR: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN); return -1; } if (len > ENCRYPT_KEY_LEN) { - printf("Error: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); + printf("ERROR: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN); return -1; } tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN); @@ -258,17 +258,12 @@ static void dmPrintArgs(int32_t argc, char const *argv[]) { static void dmGenerateGrant() { mndGenerateMachineCode(); } static void dmPrintVersion() { + printf("%s\ntaosd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, version, compatible_version); + printf("git: %s\n", gitinfo); #ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; + printf("git: %s\n", gitinfoOfInternal); #endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); -#ifdef TD_ENTERPRISE - printf("gitinfoOfInternal: %s\n", gitinfoOfInternal); -#endif - printf("buildInfo: %s\n", buildinfo); + printf("build: %s\n", buildinfo); } static void dmPrintHelp() { @@ -371,6 +366,22 @@ int mainWindows(int argc, char **argv) { printf("memory dbg enabled\n"); } #endif + if(global.generateCode) { + bool toLogFile = false; + if(taosReadDataFolder(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs) != 0){ + encryptError("failed to generate encrypt code since taosd is running, please stop it first"); + return -1; + }; + + if(dmCheckRunning(tsDataDir) == NULL) { + encryptError("failed to generate encrypt code since taosd is running, please stop it first"); + return -1; + } + int ret = dmUpdateEncryptKey(global.encryptKey, toLogFile); + taosCloseLog(); + taosCleanupArgs(); + return ret; + } if (dmInitLog() != 0) { printf("failed to start since init log error\n"); @@ -380,35 +391,13 @@ int mainWindows(int argc, char **argv) { dmPrintArgs(argc, argv); - bool isDumpCfg = true; - if(global.generateCode) { - isDumpCfg = false; - } - if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0, isDumpCfg) != 0) { + if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { dError("failed to start since read config error"); taosCloseLog(); taosCleanupArgs(); return -1; } - if(global.generateCode) { - if(dmCheckRunning(tsDataDir) == NULL) { - dError("failed to generate encrypt code since taosd is running, please stop it first"); - return -1; - } - int ret = dmUpdateEncryptKey(global.encryptKey); - taosCloseLog(); - taosCleanupArgs(); - return ret; - } - - if(dmGetEncryptKey() != 0){ - dError("failed to start since failed to get encrypt key"); - taosCloseLog(); - taosCleanupArgs(); - return -1; - }; - if (taosConvInit() != 0) { dError("failed to init conv"); taosCloseLog(); @@ -446,6 +435,13 @@ int mainWindows(int argc, char **argv) { osSetProcPath(argc, (char **)argv); taosCleanupArgs(); + if(dmGetEncryptKey() != 0){ + dError("failed to start since failed to get encrypt key"); + taosCloseLog(); + taosCleanupArgs(); + return -1; + }; + if (dmInit() != 0) { if (terrno == TSDB_CODE_NOT_FOUND) { dError("failed to init dnode since unsupported platform, please visit https://www.taosdata.com for support"); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3ec080fb21..800f7f7864 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -220,7 +220,7 @@ int32_t dmProcessCreateEncryptKeyReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _exit; } - code = dmUpdateEncryptKey(cfgReq.value); + code = dmUpdateEncryptKey(cfgReq.value, true); if (code == 0) { tsEncryptionKeyChksum = taosCalcChecksum(0, cfgReq.value, strlen(cfgReq.value)); tsEncryptionKeyStat = ENCRYPT_KEY_STAT_LOADED; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 880e96adfb..9b07b6a3d8 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -78,6 +78,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; @@ -88,6 +89,8 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0295e88106..bfc9e92293 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -961,12 +961,16 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 8e760c28be..5b1f31e6c6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -415,3 +415,7 @@ void dmReportStartup(const char *pName, const char *pDesc) { } int64_t dmGetClusterId() { return globalDnode.data.clusterId; } + +bool dmReadyForTest() { + return dmInstance()->data.dnodeVer > 0; +} diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 9fdd5e50ed..aea3286d76 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -53,6 +53,36 @@ extern "C" { #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} +#define encryptDebug(...) { \ + if (toLogFile) { \ + if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__);} \ + } else { \ + /*if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_SCREEN, dDebugFlag, __VA_ARGS__);}*/ \ + if (dDebugFlag & DEBUG_DEBUG) {printf(__VA_ARGS__); printf("\n");} \ + } \ +} + +#define encryptInfo(...) { \ + if (toLogFile) { \ + taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); \ + } else { \ + /*if (dDebugFlag & DEBUG_DEBUG) {taosPrintLog("DND ", DEBUG_SCREEN, dDebugFlag, __VA_ARGS__);}*/ \ + printf(__VA_ARGS__); \ + printf("\n"); \ + } \ +} + +#define encryptError(...) { \ + if (toLogFile) { \ + taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \ + }\ + else{ \ + /*taosPrintLog("DND ", DEBUG_SCREEN, 255, __VA_ARGS__); */\ + printf("ERROR: " __VA_ARGS__); \ + printf("\n"); \ + }\ +} + #define dGFatal(param, ...) {if (dDebugFlag & DEBUG_FATAL) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGError(param, ...) {if (dDebugFlag & DEBUG_ERROR) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}} #define dGWarn(param, ...) {if (dDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn(param ", gtid:%s", __VA_ARGS__, buf);}} @@ -194,7 +224,7 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); void dmRemoveDnodePairs(SDnodeData *pData); void dmGetDnodeEp(void *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort); -int32_t dmUpdateEncryptKey(char *key); +int32_t dmUpdateEncryptKey(char *key, bool toLogFile); int32_t dmGetEncryptKey(); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index e2c52cfb7e..752abb83a2 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -186,7 +186,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) { extern int32_t generateEncryptCode(const char *key, const char *machineId, char **encryptCode); -static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){ +static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key, bool toLogFile){ TdFilePtr pFile = NULL; char *result = NULL; int32_t code = -1; @@ -211,7 +211,8 @@ static int32_t dmWriteCheckCodeFile(char* file, char* realfile, char* key){ taosCloseFile(&pFile); if (taosRenameFile(file, realfile) != 0) goto _OVER; - dInfo("succeed to write checkCode file:%s", realfile); + encryptDebug("succeed to write checkCode file:%s", realfile); + code = 0; _OVER: if(pFile != NULL) taosCloseFile(&pFile); @@ -220,7 +221,7 @@ _OVER: return code; } -static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode){ +static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptCode, bool toLogFile){ TdFilePtr pFile = NULL; int32_t code = -1; @@ -234,7 +235,7 @@ static int32_t dmWriteEncryptCodeFile(char* file, char* realfile, char* encryptC taosCloseFile(&pFile); if (taosRenameFile(file, realfile) != 0) goto _OVER; - dInfo("succeed to write encryptCode file:%s", realfile); + encryptDebug("succeed to write encryptCode file:%s", realfile); code = 0; _OVER: @@ -243,7 +244,7 @@ _OVER: return code; } -static int32_t dmCompareEncryptKey(char* file, char* key){ +static int32_t dmCompareEncryptKey(char* file, char* key, bool toLogFile){ char *content = NULL; int64_t size = 0; TdFilePtr pFile = NULL; @@ -253,13 +254,13 @@ static int32_t dmCompareEncryptKey(char* file, char* key){ pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to open dnode file:%s since %s", file, terrstr()); + encryptError("failed to open dnode file:%s since %s", file, terrstr()); goto _OVER; } if (taosFStatFile(pFile, &size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to fstat dnode file:%s since %s", file, terrstr()); + encryptError("failed to fstat dnode file:%s since %s", file, terrstr()); goto _OVER; } @@ -271,11 +272,11 @@ static int32_t dmCompareEncryptKey(char* file, char* key){ if (taosReadFile(pFile, content, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to read dnode file:%s since %s", file, terrstr()); + encryptError("failed to read dnode file:%s since %s", file, terrstr()); goto _OVER; } - dInfo("succeed to read checkCode file:%s", file); + encryptDebug("succeed to read checkCode file:%s", file); int len = ENCRYPTED_LEN(size); result = taosMemoryMalloc(len); @@ -290,11 +291,11 @@ static int32_t dmCompareEncryptKey(char* file, char* key){ if(strcmp(opts.result, DM_KEY_INDICATOR) != 0) { terrno = TSDB_CODE_DNODE_ENCRYPTKEY_CHANGED; - dError("failed to compare decrypted result"); + encryptError("failed to compare decrypted result"); goto _OVER; } - dInfo("succeed to compare checkCode file:%s", file); + encryptDebug("succeed to compare checkCode file:%s", file); code = 0; _OVER: if(result != NULL) taosMemoryFree(result); @@ -304,7 +305,7 @@ _OVER: return code; } -int32_t dmUpdateEncryptKey(char *key) { +int32_t dmUpdateEncryptKey(char *key, bool toLogFile) { #ifdef TD_ENTERPRISE int32_t code = -1; char *machineId = NULL; @@ -328,12 +329,12 @@ int32_t dmUpdateEncryptKey(char *key) { if (taosMkDir(folder) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", folder, terrstr()); + encryptError("failed to create dir:%s since %s", folder, terrstr()); goto _OVER; } if(taosCheckExistFile(realCheckFile)){ - if(dmCompareEncryptKey(realCheckFile, key) != 0){ + if(dmCompareEncryptKey(realCheckFile, key, toLogFile) != 0){ goto _OVER; } } @@ -347,21 +348,23 @@ int32_t dmUpdateEncryptKey(char *key) { goto _OVER; } - if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode) != 0){ + if(dmWriteEncryptCodeFile(encryptFile, realEncryptFile, encryptCode, toLogFile) != 0){ goto _OVER; } - if(dmWriteCheckCodeFile(checkFile, realCheckFile, key) != 0){ + if(dmWriteCheckCodeFile(checkFile, realCheckFile, key, toLogFile) != 0){ goto _OVER; } + encryptInfo("Succeed to update encrypt key\n"); + code = 0; _OVER: taosMemoryFree(encryptCode); taosMemoryFree(machineId); if (code != 0) { if (terrno == 0) terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to update encrypt key since %s", terrstr()); + encryptError("failed to update encrypt key since %s", terrstr()); } return code; #else @@ -453,7 +456,7 @@ int32_t dmGetEncryptKey(){ goto _OVER; } - if(dmCompareEncryptKey(checkFile, encryptKey) != 0){ + if(dmCompareEncryptKey(checkFile, encryptKey, true) != 0){ goto _OVER; } diff --git a/source/dnode/mgmt/test/sut/inc/server.h b/source/dnode/mgmt/test/sut/inc/server.h index 7343276210..78055393b5 100644 --- a/source/dnode/mgmt/test/sut/inc/server.h +++ b/source/dnode/mgmt/test/sut/inc/server.h @@ -20,10 +20,10 @@ class TestServer { public: bool Start(); void Stop(); -bool runnning; + bool running; private: TdThread threadId; }; -#endif /* _TD_TEST_SERVER_H_ */ \ No newline at end of file +#endif /* _TD_TEST_SERVER_H_ */ diff --git a/source/dnode/mgmt/test/sut/src/server.cpp b/source/dnode/mgmt/test/sut/src/server.cpp index 81e6dcf495..2218504df4 100644 --- a/source/dnode/mgmt/test/sut/src/server.cpp +++ b/source/dnode/mgmt/test/sut/src/server.cpp @@ -17,13 +17,11 @@ void* serverLoop(void* param) { TestServer* server = (TestServer*)param; - server->runnning = false; if (dmInit() != 0) { return NULL; } - server->runnning = true; if (dmRun() != 0) { return NULL; } @@ -33,13 +31,18 @@ void* serverLoop(void* param) { } bool TestServer::Start() { + tstrncpy(tsVersionName, "trial", strlen("trial")); + running = false; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); taosThreadCreate(&threadId, &thAttr, serverLoop, this); taosThreadAttrDestroy(&thAttr); - taosMsleep(2100); - return runnning; + while (!dmReadyForTest()) { + taosMsleep(500); + } + running = true; + return running; } void TestServer::Stop() { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 68c55e235f..5c21e9b22b 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -180,7 +180,7 @@ typedef struct { tmsg_t originRpcType; char dbname[TSDB_TABLE_FNAME_LEN]; char stbname[TSDB_TABLE_FNAME_LEN]; - int32_t arbGroupId; + SHashObj* arbGroupIds; int32_t startFunc; int32_t stopFunc; int32_t paramLen; @@ -255,6 +255,7 @@ typedef struct { typedef struct { int32_t dnodeId; char token[TSDB_ARB_TOKEN_SIZE]; + int8_t acked; } SArbAssignedLeader; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 5a4caf3348..6aed50e508 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -46,18 +46,8 @@ typedef struct SVgroupChangeInfo { SArray *pUpdateNodeList; // SArray } SVgroupChangeInfo; -// time to generated the checkpoint, if now() - checkpointTs >= tsCheckpointInterval, this checkpoint will be discard -// to avoid too many checkpoints for a taskk in the waiting list -typedef struct SCheckpointCandEntry { - char *pName; - int64_t streamId; - int64_t checkpointTs; - int64_t checkpointId; -} SCheckpointCandEntry; - typedef struct SStreamTransMgmt { SHashObj *pDBTrans; - SHashObj *pWaitingList; // stream id list, of which timed checkpoint failed to be issued due to the trans conflict. } SStreamTransMgmt; typedef struct SStreamExecInfo { @@ -97,7 +87,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(STrans *pTrans, SStreamObj *pStream); int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId); -int32_t mndAddtoCheckpointWaitingList(SStreamObj *pStream, int64_t checkpointId); +int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt); bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock); int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId); @@ -116,8 +106,8 @@ int32_t mndStreamSetUpdateEpsetAction(SMnode *pMnode, SStreamObj *pStream, SVgr SStreamObj *mndGetStreamObj(SMnode *pMnode, int64_t streamId); int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool *hasEpset, int32_t taskId, int32_t nodeId); int32_t mndProcessStreamHb(SRpcMsg *pReq); -void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode); -int32_t initStreamNodeList(SMnode *pMnode); +void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); +int32_t extractStreamNodeList(SMnode *pMnode); int32_t mndStreamSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamObj *pStream, int8_t igUntreated); int32_t mndStreamSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); @@ -130,6 +120,8 @@ void destroyStreamTaskIter(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter); SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); void mndInitExecInfo(); +void removeExpiredNodeInfo(const SArray *pNodeSnapshot); +void removeTasksInBuf(SArray* pTaskIds, SStreamExecInfo* pExecInfo); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 8c9ca87fb1..8008eb76e7 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -78,7 +78,7 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void *param, int32_t paramLen); void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); -void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId); +void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId); void mndTransSetSerial(STrans *pTrans); void mndTransSetParallel(STrans *pTrans); void mndTransSetChangeless(STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndArbGroup.c b/source/dnode/mnode/impl/src/mndArbGroup.c index d0a86bdde7..6a6b3d2daa 100644 --- a/source/dnode/mnode/impl/src/mndArbGroup.c +++ b/source/dnode/mnode/impl/src/mndArbGroup.c @@ -25,7 +25,7 @@ #include "mndVgroup.h" #define ARBGROUP_VER_NUMBER 1 -#define ARBGROUP_RESERVE_SIZE 64 +#define ARBGROUP_RESERVE_SIZE 63 static SHashObj *arbUpdateHash = NULL; @@ -39,10 +39,11 @@ static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup); static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew); static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup); +static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray); static int32_t mndProcessArbHbTimer(SRpcMsg *pReq); static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq); -static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq); +static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq); static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp); static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp); static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp); @@ -68,7 +69,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_ARB_HEARTBEAT_TIMER, mndProcessArbHbTimer); mndSetMsgHandle(pMnode, TDMT_MND_ARB_CHECK_SYNC_TIMER, mndProcessArbCheckSyncTimer); - mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP, mndProcessArbUpdateGroupReq); + mndSetMsgHandle(pMnode, TDMT_MND_ARB_UPDATE_GROUP_BATCH, mndProcessArbUpdateGroupBatchReq); mndSetMsgHandle(pMnode, TDMT_VND_ARB_HEARTBEAT_RSP, mndProcessArbHbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ARB_CHECK_SYNC_RSP, mndProcessArbCheckSyncRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_SET_ASSIGNED_LEADER_RSP, mndProcessArbSetAssignedLeaderRsp); @@ -81,9 +82,7 @@ int32_t mndInitArbGroup(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -void mndCleanupArbGroup(SMnode *pMnode) { - taosHashCleanup(arbUpdateHash); -} +void mndCleanupArbGroup(SMnode *pMnode) { taosHashCleanup(arbUpdateHash); } SArbGroup *mndAcquireArbGroup(SMnode *pMnode, int32_t vgId) { SArbGroup *pGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &vgId); @@ -130,6 +129,7 @@ SSdbRaw *mndArbGroupActionEncode(SArbGroup *pGroup) { SDB_SET_INT32(pRaw, dataPos, pLeader->dnodeId, _OVER) SDB_SET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER) SDB_SET_INT64(pRaw, dataPos, pGroup->version, _OVER) + SDB_SET_INT8(pRaw, dataPos, pLeader->acked, _OVER) SDB_SET_RESERVE(pRaw, dataPos, ARBGROUP_RESERVE_SIZE, _OVER) @@ -183,6 +183,7 @@ SSdbRow *mndArbGroupActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pLeader->dnodeId, _OVER) SDB_GET_BINARY(pRaw, dataPos, pLeader->token, TSDB_ARB_TOKEN_SIZE, _OVER) SDB_GET_INT64(pRaw, dataPos, &pGroup->version, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pLeader->acked, _OVER) pGroup->mutexInited = false; @@ -236,6 +237,7 @@ static int32_t mndArbGroupActionUpdate(SSdb *pSdb, SArbGroup *pOld, SArbGroup *p pOld->isSync = pNew->isSync; pOld->assignedLeader.dnodeId = pNew->assignedLeader.dnodeId; memcpy(pOld->assignedLeader.token, pNew->assignedLeader.token, TSDB_ARB_TOKEN_SIZE); + pOld->assignedLeader.acked = pNew->assignedLeader.acked; pOld->version++; _OVER: @@ -541,6 +543,16 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { return -1; } + int64_t roleTimeMs = mndGetRoleTimeMs(pMnode); + int64_t nowMs = taosGetTimestampMs(); + if (nowMs - roleTimeMs < tsArbHeartBeatIntervalSec * 1000 * 2) { + mInfo("arb skip to check sync since mnd had just switch over, roleTime:%" PRId64 " now:%" PRId64, roleTimeMs, + nowMs); + return 0; + } + + SArray *pUpdateArray = taosArrayInit(16, sizeof(SArbGroup)); + while (1) { pIter = sdbFetch(pSdb, SDB_ARBGROUP, pIter, (void **)&pArbGroup); if (pIter == NULL) break; @@ -550,15 +562,14 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { taosThreadMutexUnlock(&pArbGroup->mutex); int32_t vgId = arbGroupDup.vgId; - int64_t nowMs = taosGetTimestampMs(); bool member0IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 0, nowMs); bool member1IsTimeout = mndCheckArbMemberHbTimeout(&arbGroupDup, 1, nowMs); SArbAssignedLeader *pAssignedLeader = &arbGroupDup.assignedLeader; int32_t currentAssignedDnodeId = pAssignedLeader->dnodeId; - // 1. has assigned && is sync => send req - if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true) { + // 1. has assigned && is sync && no response => send req + if (currentAssignedDnodeId != 0 && arbGroupDup.isSync == true && pAssignedLeader->acked == false) { (void)mndSendArbSetAssignedLeaderReq(pMnode, currentAssignedDnodeId, vgId, arbToken, term, pAssignedLeader->token); mInfo("vgId:%d, arb send set assigned leader to dnodeId:%d", vgId, currentAssignedDnodeId); @@ -612,40 +623,27 @@ static int32_t mndProcessArbCheckSyncTimer(SRpcMsg *pReq) { SArbGroup newGroup = {0}; mndArbGroupDupObj(&arbGroupDup, &newGroup); mndArbGroupSetAssignedLeader(&newGroup, candidateIndex); - if (mndPullupArbUpdateGroup(pMnode, &newGroup) != 0) { - mError("vgId:%d, arb failed to pullup set assigned leader to dnodeId:%d, since %s", vgId, pMember->info.dnodeId, - terrstr()); - sdbRelease(pSdb, pArbGroup); - return -1; - } - - mInfo("vgId:%d, arb pull up set assigned leader to dnodeId:%d", vgId, pMember->info.dnodeId); + taosArrayPush(pUpdateArray, &newGroup); sdbRelease(pSdb, pArbGroup); } + (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray); + + taosArrayDestroy(pUpdateArray); return 0; } -static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) { - SMArbUpdateGroupReq req = {0}; - req.vgId = pNewGroup->vgId; - req.dbUid = pNewGroup->dbUid; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - req.members[i].dnodeId = pNewGroup->members[i].info.dnodeId; - req.members[i].token = pNewGroup->members[i].state.token; - } - req.isSync = pNewGroup->isSync; - req.assignedLeader.dnodeId = pNewGroup->assignedLeader.dnodeId; - req.assignedLeader.token = pNewGroup->assignedLeader.token; - req.version = pNewGroup->version; +static void *mndBuildArbUpdateGroupBatchReq(int32_t *pContLen, SArray *updateArray) { + SMArbUpdateGroupBatchReq req = {0}; + req.updateArray = updateArray; - int32_t contLen = tSerializeSMArbUpdateGroupReq(NULL, 0, &req); + int32_t contLen = tSerializeSMArbUpdateGroupBatchReq(NULL, 0, &req); if (contLen <= 0) return NULL; SMsgHead *pHead = rpcMallocCont(contLen); if (pHead == NULL) return NULL; - if (tSerializeSMArbUpdateGroupReq(pHead, contLen, &req) <= 0) { + if (tSerializeSMArbUpdateGroupBatchReq(pHead, contLen, &req) <= 0) { rpcFreeCont(pHead); return NULL; } @@ -653,60 +651,174 @@ static void *mndBuildArbUpdateGroupReq(int32_t *pContLen, SArbGroup *pNewGroup) return pHead; } +static void mndInitArbUpdateGroup(SArbGroup *pGroup, SMArbUpdateGroup *outGroup) { + outGroup->vgId = pGroup->vgId; + outGroup->dbUid = pGroup->dbUid; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + outGroup->members[i].dnodeId = pGroup->members[i].info.dnodeId; + outGroup->members[i].token = pGroup->members[i].state.token; // just copy the pointer + } + outGroup->isSync = pGroup->isSync; + outGroup->assignedLeader.dnodeId = pGroup->assignedLeader.dnodeId; + outGroup->assignedLeader.token = pGroup->assignedLeader.token; // just copy the pointer + outGroup->assignedLeader.acked = pGroup->assignedLeader.acked; + outGroup->version = pGroup->version; +} + static int32_t mndPullupArbUpdateGroup(SMnode *pMnode, SArbGroup *pNewGroup) { if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) { mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId); return 0; } - int32_t contLen = 0; - void *pHead = mndBuildArbUpdateGroupReq(&contLen, pNewGroup); - if (!pHead) { - mError("vgId:%d, failed to build arb-update-group request", pNewGroup->vgId); - return -1; - } - SRpcMsg rpcMsg = {.msgType = TDMT_MND_ARB_UPDATE_GROUP, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + int32_t ret = -1; - int32_t ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - if (ret == 0) { - taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); + SMArbUpdateGroup newGroup = {0}; + mndInitArbUpdateGroup(pNewGroup, &newGroup); + + SArray *pArray = taosArrayInit(1, sizeof(SMArbUpdateGroup)); + taosArrayPush(pArray, &newGroup); + + int32_t contLen = 0; + void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray); + if (!pHead) { + mError("failed to build arb-update-group request"); + goto _OVER; } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + if (ret != 0) goto _OVER; + + taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); + +_OVER: + taosArrayDestroy(pArray); return ret; } -static int32_t mndProcessArbUpdateGroupReq(SRpcMsg *pReq) { - int ret = 0; +static int32_t mndPullupArbUpdateGroupBatch(SMnode *pMnode, SArray *newGroupArray) { + int32_t ret = -1; - SMArbUpdateGroupReq req = {0}; - tDeserializeSMArbUpdateGroupReq(pReq->pCont, pReq->contLen, &req); + size_t sz = taosArrayGetSize(newGroupArray); + SArray *pArray = taosArrayInit(sz, sizeof(SMArbUpdateGroup)); + for (size_t i = 0; i < sz; i++) { + SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i); + if (taosHashGet(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)) != NULL) { + mInfo("vgId:%d, arb skip to pullup arb-update-group request, since it is in process", pNewGroup->vgId); + continue; + } - SArbGroup newGroup = {0}; - newGroup.vgId = req.vgId; - newGroup.dbUid = req.dbUid; - for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { - newGroup.members[i].info.dnodeId = req.members[i].dnodeId; - memcpy(newGroup.members[i].state.token, req.members[i].token, TSDB_ARB_TOKEN_SIZE); + SMArbUpdateGroup newGroup = {0}; + mndInitArbUpdateGroup(pNewGroup, &newGroup); + + taosArrayPush(pArray, &newGroup); + taosHashPut(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId), NULL, 0); } - newGroup.isSync = req.isSync; - newGroup.assignedLeader.dnodeId = req.assignedLeader.dnodeId; - memcpy(newGroup.assignedLeader.token, req.assignedLeader.token, TSDB_ARB_TOKEN_SIZE); - newGroup.version = req.version; - - SMnode *pMnode = pReq->info.node; - SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId); - if (!pOldGroup) { - mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId); - return 0; - } - sdbRelease(pMnode->pSdb, pOldGroup); - - if (mndArbGroupUpdateTrans(pMnode, &newGroup) != 0) { - mError("vgId:%d, arb failed to update arbgroup, since %s", newGroup.vgId, terrstr()); - ret = -1; + if (taosArrayGetSize(pArray) == 0) { + ret = 0; + goto _OVER; } - tFreeSMArbUpdateGroupReq(&req); + int32_t contLen = 0; + void *pHead = mndBuildArbUpdateGroupBatchReq(&contLen, pArray); + if (!pHead) { + mError("failed to build arb-update-group request"); + goto _OVER; + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_ARB_UPDATE_GROUP_BATCH, .pCont = pHead, .contLen = contLen, .info.noResp = true}; + ret = tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + +_OVER: + taosArrayDestroy(pArray); + + if (ret != 0) { + for (size_t i = 0; i < sz; i++) { + SArbGroup *pNewGroup = taosArrayGet(newGroupArray, i); + taosHashRemove(arbUpdateHash, &pNewGroup->vgId, sizeof(pNewGroup->vgId)); + } + } + + return ret; +} + +static int32_t mndProcessArbUpdateGroupBatchReq(SRpcMsg *pReq) { + int ret = -1; + size_t sz = 0; + + SMArbUpdateGroupBatchReq req = {0}; + if (tDeserializeSMArbUpdateGroupBatchReq(pReq->pCont, pReq->contLen, &req) != 0) { + mError("arb failed to decode arb-update-group request"); + return -1; + } + + SMnode *pMnode = pReq->info.node; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_ARBGROUP, NULL, "update-arbgroup"); + if (pTrans == NULL) { + mError("failed to update arbgroup in create trans, since %s", terrstr()); + goto _OVER; + } + + sz = taosArrayGetSize(req.updateArray); + for (size_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); + SArbGroup newGroup = {0}; + newGroup.vgId = pUpdateGroup->vgId; + newGroup.dbUid = pUpdateGroup->dbUid; + for (int i = 0; i < TSDB_ARB_GROUP_MEMBER_NUM; i++) { + newGroup.members[i].info.dnodeId = pUpdateGroup->members[i].dnodeId; + memcpy(newGroup.members[i].state.token, pUpdateGroup->members[i].token, TSDB_ARB_TOKEN_SIZE); + } + + newGroup.isSync = pUpdateGroup->isSync; + newGroup.assignedLeader.dnodeId = pUpdateGroup->assignedLeader.dnodeId; + memcpy(newGroup.assignedLeader.token, pUpdateGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE); + newGroup.assignedLeader.acked = pUpdateGroup->assignedLeader.acked; + newGroup.version = pUpdateGroup->version; + + SArbGroup *pOldGroup = sdbAcquire(pMnode->pSdb, SDB_ARBGROUP, &newGroup.vgId); + if (!pOldGroup) { + mInfo("vgId:%d, arb skip to update arbgroup, since no obj found", newGroup.vgId); + taosHashRemove(arbUpdateHash, &newGroup.vgId, sizeof(int32_t)); + continue; + } + + mndTransAddArbGroupId(pTrans, newGroup.vgId); + + if (mndSetCreateArbGroupCommitLogs(pTrans, &newGroup) != 0) { + mError("failed to update arbgroup in set commit log, vgId:%d, trans:%d, since %s", newGroup.vgId, pTrans->id, + terrstr()); + goto _OVER; + } + + mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]", + pTrans->id, newGroup.vgId, newGroup.members[0].info.dnodeId, newGroup.members[0].state.token, + newGroup.members[1].info.dnodeId, newGroup.members[1].state.token, newGroup.isSync, + newGroup.assignedLeader.dnodeId, newGroup.assignedLeader.token, newGroup.assignedLeader.acked); + + sdbRelease(pMnode->pSdb, pOldGroup); + } + + if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + ret = 0; + +_OVER: + if (ret != 0) { + // failed to update arbgroup + for (size_t i = 0; i < sz; i++) { + SMArbUpdateGroup *pUpdateGroup = taosArrayGet(req.updateArray, i); + taosHashRemove(arbUpdateHash, &pUpdateGroup->vgId, sizeof(int32_t)); + } + } + + mndTransDrop(pTrans); + tFreeSMArbUpdateGroupBatchReq(&req); return ret; } @@ -719,11 +831,13 @@ static void mndArbGroupSetAssignedLeader(SArbGroup *pGroup, int32_t index) { pGroup->assignedLeader.dnodeId = pMember->info.dnodeId; strncpy(pGroup->assignedLeader.token, pMember->state.token, TSDB_ARB_TOKEN_SIZE); + pGroup->assignedLeader.acked = false; } static void mndArbGroupResetAssignedLeader(SArbGroup *pGroup) { pGroup->assignedLeader.dnodeId = 0; memset(pGroup->assignedLeader.token, 0, TSDB_ARB_TOKEN_SIZE); + pGroup->assignedLeader.acked = false; } static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { @@ -734,12 +848,12 @@ static int32_t mndArbGroupUpdateTrans(SMnode *pMnode, SArbGroup *pNew) { goto _OVER; } - mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s]", + mInfo("trans:%d, used to update arbgroup:%d, member0:[%d][%s] member1:[%d][%s] isSync:%d assigned:[%d][%s][%d]", pTrans->id, pNew->vgId, pNew->members[0].info.dnodeId, pNew->members[0].state.token, pNew->members[1].info.dnodeId, pNew->members[1].state.token, pNew->isSync, pNew->assignedLeader.dnodeId, - pNew->assignedLeader.token); + pNew->assignedLeader.token, pNew->assignedLeader.acked); - mndTransSetArbGroupId(pTrans, pNew->vgId); + mndTransAddArbGroupId(pTrans, pNew->vgId); if (mndTransCheckConflict(pMnode, pTrans) != 0) { ret = -1; goto _OVER; @@ -816,10 +930,10 @@ _OVER: } static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *memberArray) { - int ret = 0; int64_t nowMs = taosGetTimestampMs(); + size_t size = taosArrayGetSize(memberArray); + SArray *pUpdateArray = taosArrayInit(size, sizeof(SArbGroup)); - size_t size = taosArrayGetSize(memberArray); for (size_t i = 0; i < size; i++) { SVArbHbRspMember *pRspMember = taosArrayGet(memberArray, i); @@ -832,17 +946,16 @@ static int32_t mndUpdateArbHeartBeat(SMnode *pMnode, int32_t dnodeId, SArray *me bool updateToken = mndUpdateArbGroupByHeartBeat(pGroup, pRspMember, nowMs, dnodeId, &newGroup); if (updateToken) { - ret = mndPullupArbUpdateGroup(pMnode, &newGroup); - if (ret != 0) { - mInfo("failed to pullup update arb token, vgId:%d, since %s", pRspMember->vgId, terrstr()); - } + taosArrayPush(pUpdateArray, &newGroup); } sdbRelease(pMnode->pSdb, pGroup); - if (ret != 0) break; } - return ret; + (void)mndPullupArbUpdateGroupBatch(pMnode, pUpdateArray); + + taosArrayDestroy(pUpdateArray); + return 0; } bool mndUpdateArbGroupByCheckSync(SArbGroup *pGroup, int32_t vgId, char *member0Token, char *member1Token, @@ -900,6 +1013,11 @@ static int32_t mndUpdateArbSync(SMnode *pMnode, int32_t vgId, char *member0Token } static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb hb-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -914,6 +1032,7 @@ static int32_t mndProcessArbHbRsp(SRpcMsg *pRsp) { SVArbHeartBeatRsp arbHbRsp = {0}; if (tDeserializeSVArbHeartBeatRsp(pRsp->pCont, pRsp->contLen, &arbHbRsp) != 0) { + mInfo("arb hb-rsp des failed, since:%s", tstrerror(pRsp->code)); terrno = TSDB_CODE_INVALID_MSG; return -1; } @@ -934,6 +1053,11 @@ _OVER: } static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb check-sync-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -948,7 +1072,7 @@ static int32_t mndProcessArbCheckSyncRsp(SRpcMsg *pRsp) { SVArbCheckSyncRsp syncRsp = {0}; if (tDeserializeSVArbCheckSyncRsp(pRsp->pCont, pRsp->contLen, &syncRsp) != 0) { - mInfo("arb sync check failed, since:%s", tstrerror(pRsp->code)); + mInfo("arb check-sync-rsp des failed, since:%s", tstrerror(pRsp->code)); if (pRsp->code == TSDB_CODE_MND_ARB_TOKEN_MISMATCH) { terrno = TSDB_CODE_SUCCESS; return 0; @@ -993,11 +1117,12 @@ bool mndUpdateArbGroupBySetAssignedLeader(SArbGroup *pGroup, int32_t vgId, char goto _OVER; } - if (pGroup->isSync) { + if (pGroup->assignedLeader.acked == false) { mndArbGroupDupObj(pGroup, pNewGroup); pNewGroup->isSync = false; + pNewGroup->assignedLeader.acked = true; - mInfo("vgId:%d, arb isSync is setting to false", vgId); + mInfo("vgId:%d, arb received assigned ack", vgId); updateAssigned = true; goto _OVER; } @@ -1008,6 +1133,11 @@ _OVER: } static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) { + if (pRsp->contLen == 0) { + mDebug("arb set-assigned-rsp contLen is 0"); + return 0; + } + int32_t ret = -1; SMnode *pMnode = pRsp->info.node; @@ -1022,8 +1152,8 @@ static int32_t mndProcessArbSetAssignedLeaderRsp(SRpcMsg *pRsp) { SVArbSetAssignedLeaderRsp setAssignedRsp = {0}; if (tDeserializeSVArbSetAssignedLeaderRsp(pRsp->pCont, pRsp->contLen, &setAssignedRsp) != 0) { + mInfo("arb set-assigned-rsp des failed, since:%s", tstrerror(pRsp->code)); terrno = TSDB_CODE_INVALID_MSG; - mInfo("arb set assigned failed, des failed since:%s", tstrerror(pRsp->code)); return -1; } @@ -1102,12 +1232,18 @@ static int32_t mndRetrieveArbGroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock STR_WITH_MAXSIZE_TO_VARSTR(token, pGroup->assignedLeader.token, TSDB_ARB_TOKEN_SIZE + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)token, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pGroup->assignedLeader.acked, false); } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); } taosThreadMutexUnlock(&pGroup->mutex); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index c7ae941389..3eef2afcc1 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -662,6 +662,14 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SMqConsumerObj *pConsumerNew = NULL; STrans *pTrans = NULL; + if(taosArrayGetSize(subscribe.topicNames) == 0){ + SMqConsumerObj *pConsumerTmp = mndAcquireConsumer(pMnode, subscribe.consumerId); + if(pConsumerTmp == NULL){ + goto _over; + } + mndReleaseConsumer(pMnode, pConsumerTmp); + } + code = checkAndSortTopic(pMnode, subscribe.topicNames); if(code != TSDB_CODE_SUCCESS){ goto _over; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f254fb16a5..ad596289de 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -821,8 +821,7 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) { #ifdef TD_ENTERPRISE if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit; - if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) { - code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED; + if ((code = grantCheck(TSDB_GRANT_DB_ENCRYPTION)) != 0) { goto _exit; } if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) { @@ -1226,7 +1225,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (terrno != 0) code = terrno; - mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); + mError("db:%s, failed to alter since %s", alterReq.db, tstrerror(code)); } mndReleaseDb(pMnode, pDb); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 8dae4b3c11..fa4ab89875 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -36,7 +36,7 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl if (pShow->numOfRows < 1) { cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); - const char *src = "community"; + const char *src = TD_PRODUCT_NAME; STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataSetVal(pColInfo, numOfRows, tmp, false); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a78edcb05e..cad8c6d745 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -90,21 +90,6 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) { - SMStreamTickReq timerReq = { - .tick = sec, - }; - - int32_t contLen = tSerializeSMStreamTickMsg(NULL, 0, &timerReq); - if (contLen <= 0) return NULL; - void *pReq = rpcMallocCont(contLen); - if (pReq == NULL) return NULL; - - tSerializeSMStreamTickMsg(pReq, contLen, &timerReq); - *pContLen = contLen; - return pReq; -} - static void mndPullupTrans(SMnode *pMnode) { mTrace("pullup trans msg"); int32_t contLen = 0; @@ -174,21 +159,12 @@ static void mndCalMqRebalance(SMnode *pMnode) { } } -static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { - int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, sec); - if (pReq != NULL) { - SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); - } -} - -static void mndStreamCheckpointRemain(SMnode *pMnode) { - int32_t contLen = 0; - void *pReq = mndBuildCheckpointTickMsg(&contLen, 0); - if (pReq != NULL) { - SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); +static void mndStreamCheckpointTimer(SMnode *pMnode) { + SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); + if (pMsg != NULL) { + int32_t size = sizeof(SMStreamDoCheckpointMsg); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } } @@ -334,6 +310,8 @@ static int32_t minCronTime() { min = TMIN(min, tsStreamCheckpointInterval); min = TMIN(min, 6); // checkpointRemain min = TMIN(min, tsStreamNodeCheckInterval); + min = TMIN(min, tsArbHeartBeatIntervalSec); + min = TMIN(min, tsArbCheckSyncIntervalSec); int64_t telemInt = TMIN(60, (tsTelemInterval - 1)); min = TMIN(min, telemInt); @@ -367,12 +345,8 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { mndCalMqRebalance(pMnode); } - if (sec % tsStreamCheckpointInterval == 0) { - mndStreamCheckpointTick(pMnode, sec); - } - - if (sec % 5 == 0) { - mndStreamCheckpointRemain(pMnode); + if (sec % 30 == 0) { // send the checkpoint info every 30 sec + mndStreamCheckpointTimer(pMnode); } if (sec % tsStreamNodeCheckInterval == 0) { @@ -390,6 +364,18 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) { if (sec % tsUptimeInterval == 0) { mndIncreaseUpTime(pMnode); } + + if (sec % (tsArbHeartBeatIntervalSec) == 0) { + if (mndPullupArbHeartbeat(pMnode) != 0) { + mError("failed to pullup arb heartbeat, since:%s", terrstr()); + } + } + + if (sec % (tsArbCheckSyncIntervalSec) == 0) { + if (mndPullupArbCheckSync(pMnode) != 0) { + mError("failed to pullup arb check sync, since:%s", terrstr()); + } + } } void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { if (sec % (tsStatusInterval * 5) == 0) { @@ -399,6 +385,7 @@ void mndDoTimerCheckTask(SMnode *pMnode, int64_t sec) { mndSyncCheckTimeout(pMnode); } } + static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -421,18 +408,6 @@ static void *mndThreadFp(void *param) { continue; } mndDoTimerPullupTask(pMnode, sec); - - if (sec % (tsArbHeartBeatIntervalSec) == 0) { - if (mndPullupArbHeartbeat(pMnode) != 0) { - mError("failed to pullup arb heartbeat, since:%s", terrstr()); - } - } - - if (sec % (tsArbCheckSyncIntervalSec) == 0) { - if (mndPullupArbCheckSync(pMnode) != 0) { - mError("failed to pullup arb check sync, since:%s", terrstr()); - } - } } return NULL; @@ -832,10 +807,9 @@ _OVER: pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_TRIM_DB_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER || pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || - pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || - pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || - pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER || - pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { + pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || + pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER || pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || + pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, pMnode->stopped, state.restored, syncStr(state.state)); return -1; @@ -1076,6 +1050,11 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { return 0; } +int64_t mndGetRoleTimeMs(SMnode *pMnode) { + SSyncState state = syncGetState(pMnode->syncMgmt.sync); + return state.roleTimeMs; +} + void mndSetRestored(SMnode *pMnode, bool restored) { if (restored) { taosThreadRwlockWrlock(&pMnode->lock); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 61fc180dc6..0d3affe6e5 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1779,7 +1779,8 @@ static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj * return 0; } -static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) { +static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols, + int8_t withCompress) { if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) { terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; return -1; @@ -1806,29 +1807,53 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } for (int32_t i = 0; i < ncols; i++) { - SField *pField = taosArrayGet(pFields, i); - if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; - return -1; + if (withCompress) { + SFieldWithOptions *pField = taosArrayGet(pFields, i); + if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { + terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + return -1; + } + + if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { + terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; + return -1; + } + + SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; + pSchema->bytes = pField->bytes; + pSchema->type = pField->type; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = pNew->nextColId; + pNew->nextColId++; + + SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i]; + pCmpr->id = pSchema->colId; + pCmpr->alg = pField->compress; + mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); + } else { + SField *pField = taosArrayGet(pFields, i); + if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { + terrno = TSDB_CODE_MND_COLUMN_ALREADY_EXIST; + return -1; + } + + if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { + terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; + return -1; + } + + SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; + pSchema->bytes = pField->bytes; + pSchema->type = pField->type; + memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); + pSchema->colId = pNew->nextColId; + pNew->nextColId++; + + SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i]; + pCmpr->id = pSchema->colId; + pCmpr->alg = createDefaultColCmprByType(pSchema->type); + mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); } - - if (mndFindSuperTableTagIndex(pOld, pField->name) >= 0) { - terrno = TSDB_CODE_MND_TAG_ALREADY_EXIST; - return -1; - } - - SSchema *pSchema = &pNew->pColumns[pOld->numOfColumns + i]; - pSchema->bytes = pField->bytes; - pSchema->type = pField->type; - memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); - pSchema->colId = pNew->nextColId; - pNew->nextColId++; - - SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i]; - pCmpr->id = pSchema->colId; - pCmpr->alg = createDefaultColCmprByType(pSchema->type); - - mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); } pNew->colVer++; @@ -2461,7 +2486,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p code = mndAlterStbTagBytes(pMnode, pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_ADD_COLUMN: - code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); + code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 0); break; case TSDB_ALTER_TABLE_DROP_COLUMN: pField0 = taosArrayGet(pAlter->pFields, 0); @@ -2478,6 +2503,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS: code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); break; + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: + code = mndAddSuperTableColumn(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields, 1); + break; default: needRsp = false; terrno = TSDB_CODE_OPS_NOT_SUPPORT; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fb15e4b857..5fc6e465ad 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -45,9 +45,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq); static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq); -static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); -static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); -static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq); +static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -58,13 +56,13 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); -static SArray *extractNodeListFromStream(SMnode *pMnode); +static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); -static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); -static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); +static int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); @@ -114,10 +112,8 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, mndProcessStreamCheckpointInCandid); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq); @@ -143,9 +139,9 @@ int32_t mndInitStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) { taosArrayDestroy(execInfo.pTaskList); + taosArrayDestroy(execInfo.pNodeList); taosHashCleanup(execInfo.pTaskMap); taosHashCleanup(execInfo.transMgmt.pDBTrans); - taosHashCleanup(execInfo.transMgmt.pWaitingList); taosHashCleanup(execInfo.pTransferStateStreams); taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); @@ -697,6 +693,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SStreamObj streamObj = {0}; char *sql = NULL; int32_t sqlLen = 0; + const char* pMsg = "create stream tasks on dnodes"; + terrno = TSDB_CODE_SUCCESS; SCMCreateStreamReq createReq = {0}; @@ -709,8 +707,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_INVALID_PLATFORM; goto _OVER; #endif - mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql); + mInfo("stream:%s, start to create stream, sql:%s", createReq.name, createReq.sql); if (mndCheckCreateStreamReq(&createReq) != 0) { mError("stream:%s, failed to create since %s", createReq.name, terrstr()); goto _OVER; @@ -750,8 +748,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - STrans *pTrans = - doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes"); + STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, pMsg); if (pTrans == NULL) { goto _OVER; } @@ -794,8 +791,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { // add into buffer firstly // to make sure when the hb from vnode arrived, the newly created tasks have been in the task map already. taosThreadMutexLock(&execInfo.lock); - mDebug("stream stream:%s start to register tasks into task_node_list", createReq.name); - saveStreamTasksInfo(&streamObj, &execInfo); + mDebug("stream stream:%s start to register tasks into task nodeList", createReq.name); + saveTaskAndNodeInfoIntoBuf(&streamObj, &execInfo); taosThreadMutexUnlock(&execInfo.lock); // execute creation @@ -818,7 +815,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, sql, sqlLen); } else { char detail[1000] = {0}; - sprintf(detail, "dbname:%s, stream name:%s", dbname.dbname, name.dbname); + snprintf(detail, tListLen(detail), "dbname:%s, stream name:%s", dbname.dbname, name.dbname); auditRecord(pReq, pMnode->clusterId, "createStream", dbname.dbname, name.dbname, detail, strlen(detail)); } @@ -830,6 +827,7 @@ _OVER: mndReleaseStream(pMnode, pStream); tFreeSCMCreateStreamReq(&createReq); tFreeStreamObj(&streamObj); + if (sql != NULL) { taosMemoryFreeClear(sql); } @@ -842,6 +840,7 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) { void *pIter = NULL; SSdb *pSdb = pMnode->pSdb; int64_t maxChkptId = 0; + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; @@ -886,26 +885,10 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) { } } - mDebug("generated checkpoint %" PRId64 "", maxChkptId + 1); + mDebug("generate new checkpointId:%" PRId64, maxChkptId + 1); return maxChkptId + 1; } -static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { - return 0; - } - - SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); - pMsg->checkpointId = mndStreamGenChkptId(pMnode, true); - - int32_t size = sizeof(SMStreamDoCheckpointMsg); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size}; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - return 0; -} - static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger) { SStreamCheckpointSourceReq req = {0}; @@ -987,7 +970,6 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHECKPOINT_NAME, lock); if (conflict) { - mndAddtoCheckpointWaitingList(pStream, checkpointId); mWarn("checkpoint conflict with other trans in %s, ignore the checkpoint for stream:%s %" PRIx64, pStream->sourceDb, pStream->name, pStream->uid); return -1; @@ -1051,10 +1033,9 @@ _ERR: return code; } -int32_t initStreamNodeList(SMnode *pMnode) { - if (execInfo.pNodeList == NULL || (taosArrayGetSize(execInfo.pNodeList) == 0)) { - execInfo.pNodeList = taosArrayDestroy(execInfo.pNodeList); - execInfo.pNodeList = extractNodeListFromStream(pMnode); +int32_t extractStreamNodeList(SMnode *pMnode) { + if (taosArrayGetSize(execInfo.pNodeList) == 0) { + extractNodeListFromStream(pMnode, execInfo.pNodeList); } return taosArrayGetSize(execInfo.pNodeList); @@ -1064,7 +1045,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { // check if the node update happens or not taosThreadMutexLock(&execInfo.lock); - int32_t numOfNodes = initStreamNodeList(pMnode); + int32_t numOfNodes = extractStreamNodeList(pMnode); if (numOfNodes == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); execInfo.ts = taosGetTimestampSec(); @@ -1084,29 +1065,30 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { bool allReady = true; SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); if (!allReady) { - mWarn("not all vnodes ready"); + mWarn("not all vnodes ready, quit from vnodes status check"); taosArrayDestroy(pNodeSnapshot); taosThreadMutexUnlock(&execInfo.lock); return 0; } SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot); - bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); + + bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); + taosArrayDestroy(changeInfo.pUpdateNodeList); taosHashCleanup(changeInfo.pDBMap); taosArrayDestroy(pNodeSnapshot); if (nodeUpdated) { - mDebug("stream task not ready due to node update"); + mDebug("stream tasks not ready due to node update"); } taosThreadMutexUnlock(&execInfo.lock); return nodeUpdated; } -static int32_t mndCheckNodeStatus(SMnode *pMnode) { +static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { bool ready = true; - int64_t ts = taosGetTimestampSec(); if (taskNodeIsUpdated(pMnode)) { return -1; } @@ -1114,7 +1096,7 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { taosThreadMutexLock(&execInfo.lock); if (taosArrayGetSize(execInfo.pNodeList) == 0) { mDebug("stream task node change checking done, no vgroups exist, do nothing"); - execInfo.ts = ts; + ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); } for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { @@ -1130,79 +1112,117 @@ static int32_t mndCheckNodeStatus(SMnode *pMnode) { ready = false; break; } + + if (pEntry->hTaskId != 0) { + mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s related fill-history task:0x%" PRIx64 + " exists, checkpoint not issued", + pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status), + pEntry->hTaskId); + ready = false; + break; + } } taosThreadMutexUnlock(&execInfo.lock); return ready ? 0 : -1; } -static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { +typedef struct { + int64_t streamId; + int64_t duration; +} SCheckpointInterval; + +static int32_t streamWaitComparFn(const void* p1, const void* p2) { + const SCheckpointInterval* pInt1 = p1; + const SCheckpointInterval* pInt2 = p2; + if (pInt1->duration == pInt2->duration) { + return 0; + } + + return pInt1->duration > pInt2->duration? -1:1; +} + +static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; SStreamObj *pStream = NULL; int32_t code = 0; + int32_t numOfCheckpointTrans = 0; - if ((code = mndCheckNodeStatus(pMnode)) != 0) { + if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) { return code; } - // make sure the time interval between two consecutive checkpoint trans is long enough - SMStreamDoCheckpointMsg *pMsg = (SMStreamDoCheckpointMsg *)pReq->pCont; + SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval)); + int64_t now = taosGetTimestampMs(); + while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { - code = mndProcessStreamCheckpointTrans(pMnode, pStream, pMsg->checkpointId, 1, true); - sdbRelease(pSdb, pStream); - if (code == -1) { - break; - } - } - - return code; -} - -static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - void *pIter = NULL; - int32_t code = 0; - - taosThreadMutexLock(&execInfo.lock); - int32_t num = taosHashGetSize(execInfo.transMgmt.pWaitingList); - taosThreadMutexUnlock(&execInfo.lock); - if (num == 0) { - return code; - } - - if ((code = mndCheckNodeStatus(pMnode)) != 0) { - return code; - } - - SArray *pList = taosArrayInit(4, sizeof(int64_t)); - while ((pIter = taosHashIterate(execInfo.transMgmt.pWaitingList, pIter)) != NULL) { - SCheckpointCandEntry *pEntry = pIter; - - SStreamObj *ps = mndAcquireStream(pMnode, pEntry->pName); - if (ps == NULL) { + int64_t duration = now - pStream->checkpointFreq; + if (duration < tsStreamCheckpointInterval * 1000) { + sdbRelease(pSdb, pStream); continue; } - mDebug("start to launch checkpoint for stream:%s %" PRIx64 " in candidate list", pEntry->pName, pEntry->streamId); + SCheckpointInterval in = {.streamId = pStream->uid, .duration = duration}; + taosArrayPush(pList, &in); - code = mndProcessStreamCheckpointTrans(pMnode, ps, pEntry->checkpointId, 1, true); - mndReleaseStream(pMnode, ps); + int32_t currentSize = taosArrayGetSize(pList); + mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 + "s) beyond threshold:%d", + pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); - if (code == TSDB_CODE_SUCCESS) { - taosArrayPush(pList, &pEntry->streamId); + sdbRelease(pSdb, pStream); + } + + int32_t size = taosArrayGetSize(pList); + if (size == 0) { + taosArrayDestroy(pList); + return code; + } + + taosArraySort(pList, streamWaitComparFn); + mndStreamClearFinishedTrans(pMnode, &numOfCheckpointTrans); + int32_t numOfQual = taosArrayGetSize(pList); + + if (numOfCheckpointTrans > tsMaxConcurrentCheckpoint) { + mDebug( + "%d stream(s) checkpoint interval longer than %ds, ongoing checkpoint trans:%d reach maximum allowed:%d, new " + "checkpoint trans are not allowed, wait for 30s", + numOfQual, tsStreamCheckpointInterval, numOfCheckpointTrans, tsMaxConcurrentCheckpoint); + taosArrayDestroy(pList); + return code; + } + + int32_t capacity = tsMaxConcurrentCheckpoint - numOfCheckpointTrans; + mDebug( + "%d stream(s) checkpoint interval longer than %ds, %d ongoing checkpoint trans, %d new checkpoint trans allowed, " + "concurrent trans threshold:%d", + numOfQual, tsStreamCheckpointInterval, numOfCheckpointTrans, capacity, tsMaxConcurrentCheckpoint); + + int32_t started = 0; + int64_t checkpointId = mndStreamGenChkptId(pMnode, true); + + for (int32_t i = 0; i < numOfQual; ++i) { + SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); + + SStreamObj *p = mndGetStreamObj(pMnode, pCheckpointInfo->streamId); + if (p != NULL) { + code = mndProcessStreamCheckpointTrans(pMnode, p, checkpointId, 1, true); + sdbRelease(pSdb, p); + + if (code != -1) { + started += 1; + + if (started >= capacity) { + mDebug("already start %d new checkpoint trans, current active checkpoint trans:%d", started, + (started + numOfCheckpointTrans)); + break; + } + } } } - for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { - int64_t *pId = taosArrayGet(pList, i); - - taosHashRemove(execInfo.transMgmt.pWaitingList, pId, sizeof(*pId)); - } - - int32_t remain = taosHashGetSize(execInfo.transMgmt.pWaitingList); - mDebug("%d in candidate list generated checkpoint, remaining:%d", (int32_t)taosArrayGetSize(pList), remain); taosArrayDestroy(pList); return code; } @@ -1555,7 +1575,9 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); if (pe == NULL) { - mError("task:0x%" PRIx64 " not exists in vnode, no valid status/stage info", id.taskId); + mError("task:0x%" PRIx64 " not exists in any vnodes, streamName:%s, streamId:0x%" PRIx64 " createTs:%" PRId64 + " no valid status/stage info", + id.taskId, pStream->name, pStream->uid, pStream->createTime); return -1; } @@ -1620,15 +1642,76 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false); // input queue - char vbuf[30] = {0}; - char buf[25] = {0}; - const char *queueInfoStr = "%4.2fMiB (%5.2f%)"; - sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate); + char vbuf[40] = {0}; + char buf[38] = {0}; + const char *queueInfoStr = "%4.2f MiB (%6.2f%)"; + snprintf(buf, tListLen(buf), queueInfoStr, pe->inputQUsed, pe->inputRate); STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + // input total + const char* formatTotalMb = "%7.2f MiB"; + const char* formatTotalGb = "%7.2f GiB"; + if (pe->procsTotal < 1024) { + snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal); + } else { + snprintf(buf, tListLen(buf), formatTotalGb, pe->procsTotal / 1024); + } + + memset(vbuf, 0, tListLen(vbuf)); + STR_TO_VARSTR(vbuf, buf); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + + // process throughput + const char* formatKb = "%7.2f KiB/s"; + const char* formatMb = "%7.2f MiB/s"; + if (pe->procsThroughput < 1024) { + snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput); + } else { + snprintf(buf, tListLen(buf), formatMb, pe->procsThroughput / 1024); + } + + memset(vbuf, 0, tListLen(vbuf)); + STR_TO_VARSTR(vbuf, buf); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + + // output total + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { + colDataSetNULL(pColInfo, numOfRows); + } else { + sprintf(buf, formatTotalMb, pe->outputTotal); + memset(vbuf, 0, tListLen(vbuf)); + STR_TO_VARSTR(vbuf, buf); + + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + } + + // output throughput + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + if (pTask->info.taskLevel == TASK_LEVEL__SINK) { + colDataSetNULL(pColInfo, numOfRows); + } else { + if (pe->outputThroughput < 1024) { + snprintf(buf, tListLen(buf), formatKb, pe->outputThroughput); + } else { + snprintf(buf, tListLen(buf), formatMb, pe->outputThroughput / 1024); + } + + memset(vbuf, 0, tListLen(vbuf)); + STR_TO_VARSTR(vbuf, buf); + + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); + } + // output queue // sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate); // STR_TO_VARSTR(vbuf, buf); @@ -1638,12 +1721,14 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // info if (pTask->info.taskLevel == TASK_LEVEL__SINK) { - const char *sinkStr = "%.2fMiB"; - sprintf(buf, sinkStr, pe->sinkDataSize); + const char *sinkStr = "%.2f MiB"; + snprintf(buf, tListLen(buf), sinkStr, pe->sinkDataSize); } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { // offset info const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]"; - sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer); + snprintf(buf, tListLen(buf), offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer); + } else { + memset(buf, 0, tListLen(buf)); } STR_TO_VARSTR(vbuf, buf); @@ -1679,6 +1764,10 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false); + // checkpoint size + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + // checkpoint backup status pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, 0, true); @@ -1847,8 +1936,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) { - terrno = TSDB_CODE_GRANT_EXPIRED; + if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { return -1; } @@ -2081,7 +2169,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return 0; } -static SArray *extractNodeListFromStream(SMnode *pMnode) { +static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -2110,41 +2198,20 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) { sdbRelease(pSdb, pStream); } - SArray *plist = taosArrayInit(taosHashGetSize(pHash), sizeof(SNodeEntry)); + taosArrayClear(pNodeList); // convert to list pIter = NULL; while ((pIter = taosHashIterate(pHash, pIter)) != NULL) { SNodeEntry *pEntry = (SNodeEntry *)pIter; - taosArrayPush(plist, pEntry); + taosArrayPush(pNodeList, pEntry); char buf[256] = {0}; epsetToStr(&pEntry->epset, buf, tListLen(buf)); mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf); } + taosHashCleanup(pHash); - - return plist; -} - -static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { - void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); - if (p == NULL) { - return TSDB_CODE_SUCCESS; - } - taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); - - for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { - STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); - if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) { - taosArrayRemove(pExecNode->pTaskList, k); - - int32_t num = taosArrayGetSize(pExecNode->pTaskList); - mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)pRemovedId->taskId, num); - break; - } - } - return TSDB_CODE_SUCCESS; } @@ -2161,15 +2228,17 @@ static bool taskNodeExists(SArray *pList, int32_t nodeId) { return false; } -int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { +int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) { SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList); for (int32_t i = 0; i < numOfTask; ++i) { - STaskId *pId = taosArrayGet(execInfo.pTaskList, i); - STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + STaskId *pId = taosArrayGet(execInfo.pTaskList, i); - if (pEntry->nodeId == SNODE_HANDLE) continue; + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + if (pEntry->nodeId == SNODE_HANDLE) { + continue; + } bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { @@ -2177,32 +2246,13 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { } } - for (int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) { - STaskId *pId = taosArrayGet(pRemovedTasks, i); - doRemoveTasks(&execInfo, pId); - } + removeTasksInBuf(pRemovedTasks, &execInfo); mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks), (int32_t)taosArrayGetSize(execInfo.pTaskList)); - int32_t size = taosArrayGetSize(pNodeSnapshot); - SArray *pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); - for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) { - SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i); + removeExpiredNodeInfo(pNodeSnapshot); - for (int32_t j = 0; j < size; ++j) { - SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); - if (pEntry->nodeId == p->nodeId) { - taosArrayPush(pValidNodeEntryList, p); - break; - } - } - } - - taosArrayDestroy(execInfo.pNodeList); - execInfo.pNodeList = pValidNodeEntryList; - - mDebug("remain %d valid node entries", (int32_t)taosArrayGetSize(pValidNodeEntryList)); taosArrayDestroy(pRemovedTasks); return 0; } @@ -2223,7 +2273,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; taosThreadMutexLock(&execInfo.lock); - int32_t numOfNodes = initStreamNodeList(pMnode); + int32_t numOfNodes = extractStreamNodeList(pMnode); taosThreadMutexUnlock(&execInfo.lock); if (numOfNodes == 0) { @@ -2233,9 +2283,9 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { return 0; } - bool allVgroupsReady = true; - SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVgroupsReady); - if (!allVgroupsReady) { + bool allReady = true; + SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); + if (!allReady) { taosArrayDestroy(pNodeSnapshot); atomic_store_32(&mndNodeCheckSentinel, 0); mWarn("not all vnodes are ready, ignore the exec nodeUpdate check"); @@ -2243,31 +2293,31 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { } taosThreadMutexLock(&execInfo.lock); - removeExpirednodeEntryAndTask(pNodeSnapshot); + + removeExpiredNodeEntryAndTaskInBuf(pNodeSnapshot); SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeList, pNodeSnapshot); if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) { // kill current active checkpoint transaction, since the transaction is vnode wide. killAllCheckpointTrans(pMnode, &changeInfo); - code = mndProcessVgroupChange(pMnode, &changeInfo); // keep the new vnode snapshot if success if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { - mDebug("create trans successfully, update cached node list"); - taosArrayDestroy(execInfo.pNodeList); - execInfo.pNodeList = pNodeSnapshot; + extractNodeListFromStream(pMnode, execInfo.pNodeList); execInfo.ts = ts; + mDebug("create trans successfully, update cached node list, numOfNodes:%d", + (int)taosArrayGetSize(execInfo.pNodeList)); } else { mError("unexpected code during create nodeUpdate trans, code:%s", tstrerror(code)); - taosArrayDestroy(pNodeSnapshot); } } else { mDebug("no update found in nodeList"); - taosArrayDestroy(pNodeSnapshot); } + taosArrayDestroy(pNodeSnapshot); taosThreadMutexUnlock(&execInfo.lock); + taosArrayDestroy(changeInfo.pUpdateNodeList); taosHashCleanup(changeInfo.pDBMap); @@ -2283,15 +2333,15 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { return 0; } - SMStreamNodeCheckMsg *pMsg = rpcMallocCont(sizeof(SMStreamNodeCheckMsg)); + int32_t size = sizeof(SMStreamNodeCheckMsg); + SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = sizeof(SMStreamNodeCheckMsg)}; + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); return 0; } -void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) { +void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { SStreamTaskIter *pIter = createStreamTaskIter(pStream); while (streamTaskIterNextTask(pIter)) { SStreamTask *pTask = streamTaskIterGetCurrent(pIter); @@ -2304,39 +2354,31 @@ void saveStreamTasksInfo(SStreamObj *pStream, SStreamExecInfo *pExecNode) { taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry)); taosArrayPush(pExecNode->pTaskList, &id); - mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId, - (int32_t)taosArrayGetSize(pExecNode->pTaskList)); - } - } - destroyStreamTaskIter(pIter); -} + int32_t num = (int32_t)taosArrayGetSize(pExecNode->pTaskList); + mInfo("s-task:0x%x add into task buffer, total:%d", (int32_t)entry.id.taskId, num); -void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { - SStreamTaskIter *pIter = createStreamTaskIter(pStream); - while (streamTaskIterNextTask(pIter)) { - SStreamTask *pTask = streamTaskIterGetCurrent(pIter); - - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); - if (p != NULL) { - taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id)); - - for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { - STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); - if (pId->taskId == id.taskId && pId->streamId == id.streamId) { - taosArrayRemove(pExecNode->pTaskList, k); - - int32_t num = taosArrayGetSize(pExecNode->pTaskList); - mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)id.taskId, num); + // add the new vgroups if not added yet + bool exist = false; + for(int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) { + SNodeEntry* pEntry = taosArrayGet(pExecNode->pNodeList, j); + if (pEntry->nodeId == pTask->info.nodeId) { + exist = true; break; } } + + if (!exist) { + SNodeEntry nodeEntry = {.hbTimestamp = -1, .nodeId = pTask->info.nodeId}; + epsetAssign(&nodeEntry.epset, &pTask->info.epSet); + + taosArrayPush(pExecNode->pNodeList, &nodeEntry); + mInfo("vgId:%d added into nodeList, total:%d", nodeEntry.nodeId, (int)taosArrayGetSize(pExecNode->pNodeList)); + } } } destroyStreamTaskIter(pIter); - ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); } static void doAddTaskId(SArray *pList, int32_t taskId, int64_t uid, int32_t numOfTotal) { diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index a81a391c3d..42efb6589e 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -22,7 +22,7 @@ typedef struct SFailedCheckpointInfo { int32_t transId; } SFailedCheckpointInfo; -static void doExtractTasksFromStream(SMnode *pMnode) { +static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -33,11 +33,44 @@ static void doExtractTasksFromStream(SMnode *pMnode) { break; } - saveStreamTasksInfo(pStream, &execInfo); + saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo); sdbRelease(pSdb, pStream); } } +static void removeDroppedStreamTasksInBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) { + if (pMnode == NULL) { + return; + } + + int32_t num = taosArrayGetSize(pExecInfo->pTaskList); + + SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + SArray *pIdList = taosArrayInit(4, sizeof(STaskId)); + + for (int32_t i = 0; i < num; ++i) { + STaskId* pId = taosArrayGet(pExecInfo->pTaskList, i); + + void* p = taosHashGet(pHash, &pId->streamId, sizeof(int64_t)); + if (p != NULL) { + continue; + } + + void* pObj = mndGetStreamObj(pMnode, pId->streamId); + if (pObj != NULL) { + mndReleaseStream(pMnode, pObj); + taosHashPut(pHash, &pId->streamId, sizeof(int64_t), NULL, 0); + } else { + taosArrayPush(pIdList, pId); + } + } + + removeTasksInBuf(pIdList, &execInfo); + + taosArrayDestroy(pIdList); + taosHashCleanup(pHash); +} + static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); for (int32_t j = 0; j < numOfNodes; ++j) { @@ -131,18 +164,26 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { int32_t *pVgId = taosArrayGet(pNodeList, k); mInfo("set node expired for nodeId:%d, total:%d", *pVgId, num); + bool setFlag = false; int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); + for (int i = 0; i < numOfNodes; ++i) { SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); if (pNodeEntry->nodeId == *pVgId) { mInfo("vgId:%d expired for some stream tasks, needs update nodeEp", *pVgId); pNodeEntry->stageUpdated = true; + setFlag = true; break; } } - } + if (!setFlag) { + mError("failed to set nodeUpdate flag, nodeId:%d not exists in nodelist, update it", *pVgId); + ASSERT(0); + return TSDB_CODE_FAILED; + } + } return TSDB_CODE_SUCCESS; } @@ -222,10 +263,10 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamHbMsg req = {0}; - SArray *pFailedTasks = NULL; + SArray *pFailedChkpt = NULL; SArray *pOrphanTasks = NULL; - if (grantCheckExpire(TSDB_GRANT_STREAMS) < 0) { + if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if (suspendAllStreams(pMnode, &pReq->info) < 0) { return -1; } @@ -244,17 +285,21 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks); - pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); - pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); + pFailedChkpt = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); + pOrphanTasks = taosArrayInit(4, sizeof(SOrphanTask)); taosThreadMutexLock(&execInfo.lock); // extract stream task list if (taosHashGetSize(execInfo.pTaskMap) == 0) { - doExtractTasksFromStream(pMnode); + addAllStreamTasksIntoBuf(pMnode, &execInfo); + } else { + // the already dropped tasks may be added by hb from vnode at the time when the pTaskMap happens to be empty. + // let's drop them here. + removeDroppedStreamTasksInBuf(pMnode, &execInfo); } - initStreamNodeList(pMnode); + extractStreamNodeList(pMnode); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { @@ -282,16 +327,16 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { } } else { // task is idle for more than 50 sec. - if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { - if (!pTaskEntry->inputQChanging) { - pTaskEntry->inputQUnchangeCounter++; - } else { - pTaskEntry->inputQChanging = false; - } - } else { - pTaskEntry->inputQChanging = true; - pTaskEntry->inputQUnchangeCounter = 0; - } +// if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { +// if (!pTaskEntry->inputQChanging) { +// pTaskEntry->inputQUnchangeCounter++; +// } else { +// pTaskEntry->inputQChanging = false; +// } +// } else { +// pTaskEntry->inputQChanging = true; +// pTaskEntry->inputQUnchangeCounter = 0; +// } streamTaskStatusCopy(pTaskEntry, p); @@ -302,7 +347,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SFailedCheckpointInfo info = { .transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId}; - addIntoCheckpointList(pFailedTasks, &info); + addIntoCheckpointList(pFailedChkpt, &info); } } @@ -320,7 +365,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { // current checkpoint is failed, rollback from the checkpoint trans // kill the checkpoint trans and then set all tasks status to be normal - if (taosArrayGetSize(pFailedTasks) > 0) { + if (taosArrayGetSize(pFailedChkpt) > 0) { bool allReady = true; if (pMnode != NULL) { SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); @@ -331,8 +376,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal - for(int32_t i = 0; i < taosArrayGetSize(pFailedTasks); ++i) { - SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedTasks, i); + for(int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) { + SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedChkpt, i); mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", pInfo->checkpointId, pInfo->transId); @@ -351,7 +396,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { taosThreadMutexUnlock(&execInfo.lock); tCleanupStreamHbMsg(&req); - taosArrayDestroy(pFailedTasks); + taosArrayDestroy(pFailedChkpt); taosArrayDestroy(pOrphanTasks); { @@ -361,7 +406,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { pHead->vgId = htonl(req.vgId); tmsgSendRsp(&rsp); - pReq->info.handle = NULL; // disable auto rsp } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 74ad09c752..ff31aa0f7d 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -21,8 +21,6 @@ typedef struct SKeyInfo { int32_t keyLen; } SKeyInfo; -static int32_t clearFinishedTrans(SMnode* pMnode); - int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) { SStreamTransInfo info = { .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId}; @@ -30,41 +28,54 @@ int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t s return 0; } -int32_t clearFinishedTrans(SMnode* pMnode) { +int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) { size_t keyLen = 0; void* pIter = NULL; SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); + int32_t num = 0; while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { - SStreamTransInfo* pEntry = (SStreamTransInfo*)pIter; + SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter; // let's clear the finished trans - STrans* pTrans = mndAcquireTrans(pMnode, pEntry->transId); + STrans *pTrans = mndAcquireTrans(pMnode, pEntry->transId); if (pTrans == NULL) { - void* pKey = taosHashGetKey(pEntry, &keyLen); + void *pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; - mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, - pEntry->startTime); + mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime); taosArrayPush(pList, &info); } else { + if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { + num++; + } mndReleaseTrans(pMnode, pTrans); } } - size_t num = taosArrayGetSize(pList); - for (int32_t i = 0; i < num; ++i) { + int32_t size = taosArrayGetSize(pList); + for (int32_t i = 0; i < size; ++i) { SKeyInfo* pKey = taosArrayGet(pList, i); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); } - mDebug("clear %d finished stream-trans, remained:%d", (int32_t)num, taosHashGetSize(execInfo.transMgmt.pDBTrans)); + mDebug("clear %d finished stream-trans, remained:%d, active checkpoint trans:%d", size, + taosHashGetSize(execInfo.transMgmt.pDBTrans), num); terrno = TSDB_CODE_SUCCESS; taosArrayDestroy(pList); + + if (pNumOfActiveChkpt != NULL) { + *pNumOfActiveChkpt = num; + } + return 0; } +// * Transactions of different streams are not related. Here only check the conflict of transaction for a given stream. +// For a given stream: +// 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. +// 2. create/drop/reset/update trans are conflict with any other trans. bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) { if (lock) { taosThreadMutexLock(&execInfo.lock); @@ -78,7 +89,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p return false; } - clearFinishedTrans(pMnode); + mndStreamClearFinishedTrans(pMnode, NULL); SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { @@ -95,7 +106,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p terrno = TSDB_CODE_MND_TRANS_CONFLICT; return true; } else { - mDebug("not conflict with checkpoint trans, name:%s, continue create trans", pTransName); + mDebug("not conflict with checkpoint trans, name:%s, continue creating trans", pTransName); } } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0) || @@ -106,7 +117,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p return true; } } else { - mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamId); + mDebug("stream:0x%" PRIx64 " no conflict trans existed, continue create trans", streamId); } if (lock) { @@ -124,7 +135,7 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { return 0; } - clearFinishedTrans(pMnode); + mndStreamClearFinishedTrans(pMnode, NULL); SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; @@ -140,28 +151,6 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { return 0; } -int32_t mndAddtoCheckpointWaitingList(SStreamObj* pStream, int64_t checkpointId) { - SCheckpointCandEntry* pEntry = taosHashGet(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid)); - if (pEntry == NULL) { - SCheckpointCandEntry entry = {.streamId = pStream->uid, - .checkpointTs = taosGetTimestampMs(), - .checkpointId = checkpointId, - .pName = taosStrdup(pStream->name)}; - - taosHashPut(execInfo.transMgmt.pWaitingList, &pStream->uid, sizeof(pStream->uid), &entry, sizeof(entry)); - int32_t size = taosHashGetSize(execInfo.transMgmt.pWaitingList); - - mDebug("stream:%" PRIx64 " add into waiting list due to conflict, ts:%" PRId64 " , checkpointId: %" PRId64 - ", total in waitingList:%d", - pStream->uid, entry.checkpointTs, checkpointId, size); - } else { - mDebug("stream:%" PRIx64 " ts:%" PRId64 ", checkpointId:%" PRId64 " already in waiting list, no need to add into", - pStream->uid, pEntry->checkpointTs, checkpointId); - } - - return TSDB_CODE_SUCCESS; -} - STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index d5bc12f9df..d138254afd 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -26,6 +26,8 @@ struct SStreamTaskIter { SStreamTask *pTask; }; +int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId); + SStreamTaskIter* createStreamTaskIter(SStreamObj* pStream) { SStreamTaskIter* pIter = taosMemoryCalloc(1, sizeof(SStreamTaskIter)); if (pIter == NULL) { @@ -135,6 +137,7 @@ SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { char buf[256] = {0}; epsetToStr(&entry.epset, buf, tListLen(buf)); mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + taosArrayPush(pVgroupListSnapshot, &entry); sdbRelease(pSdb, pObj); } @@ -558,11 +561,6 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj * return 0; } -static void freeCheckpointCandEntry(void *param) { - SCheckpointCandEntry *pEntry = param; - taosMemoryFreeClear(pEntry->pName); -} - static void freeTaskList(void* param) { SArray** pList = (SArray **)param; taosArrayDestroy(*pList); @@ -575,9 +573,76 @@ void mndInitExecInfo() { execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); - execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK); execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); - taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); } + +void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { + SArray *pValidList = taosArrayInit(4, sizeof(SNodeEntry)); + int32_t size = taosArrayGetSize(pNodeSnapshot); + + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeList); ++i) { + SNodeEntry *p = taosArrayGet(execInfo.pNodeList, i); + + for (int32_t j = 0; j < size; ++j) { + SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); + if (pEntry->nodeId == p->nodeId) { + taosArrayPush(pValidList, p); + break; + } + } + } + + taosArrayDestroy(execInfo.pNodeList); + execInfo.pNodeList = pValidList; + + mDebug("remain %d valid node entries after clean expired nodes info", (int32_t)taosArrayGetSize(pValidList)); +} + +int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { + void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); + if (p == NULL) { + return TSDB_CODE_SUCCESS; + } + + taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); + + for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { + STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); + if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) { + taosArrayRemove(pExecNode->pTaskList, k); + + int32_t num = taosArrayGetSize(pExecNode->pTaskList); + mInfo("s-task:0x%x removed from buffer, remain:%d in buffer list", (int32_t)pRemovedId->taskId, num); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo* pExecInfo) { + for (int32_t i = 0; i < taosArrayGetSize(pTaskIds); ++i) { + STaskId *pId = taosArrayGet(pTaskIds, i); + doRemoveTasks(pExecInfo, pId); + } +} + +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { + taosThreadMutexLock(&pExecNode->lock); + + SStreamTaskIter *pIter = createStreamTaskIter(pStream); + while (streamTaskIterNextTask(pIter)) { + SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + doRemoveTasks(pExecNode, &id); + } + + ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); + taosThreadMutexUnlock(&pExecNode->lock); + + destroyStreamTaskIter(pIter); +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 84940e01d4..05f685ecb4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -26,7 +26,7 @@ #define TRANS_VER1_NUMBER 1 #define TRANS_VER2_NUMBER 2 #define TRANS_ARRAY_SIZE 8 -#define TRANS_RESERVE_SIZE 48 +#define TRANS_RESERVE_SIZE 44 static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); @@ -196,10 +196,21 @@ SSdbRaw *mndTransEncode(STrans *pTrans) { } SDB_SET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER) + + int32_t arbGroupNum = taosHashGetSize(pTrans->arbGroupIds); + SDB_SET_INT32(pRaw, dataPos, arbGroupNum, _OVER) + void *pIter = NULL; + pIter = taosHashIterate(pTrans->arbGroupIds, NULL); + while (pIter) { + int32_t arbGroupId = *(int32_t *)pIter; + SDB_SET_INT32(pRaw, dataPos, arbGroupId, _OVER) + pIter = taosHashIterate(pTrans->arbGroupIds, pIter); + } + SDB_SET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) - terrno = 0; + terrno = 0; _OVER: if (terrno != 0) { @@ -279,6 +290,7 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { int32_t undoActionNum = 0; int32_t commitActionNum = 0; int32_t dataPos = 0; + int32_t arbgroupIdNum = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; @@ -350,6 +362,16 @@ SSdbRow *mndTransDecode(SSdbRaw *pRaw) { } SDB_GET_BINARY(pRaw, dataPos, pTrans->opername, TSDB_TRANS_OPER_LEN, _OVER); + + pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); + + SDB_GET_INT32(pRaw, dataPos, &arbgroupIdNum, _OVER) + for (int32_t i = 0; i < arbgroupIdNum; ++i) { + int32_t arbGroupId = 0; + SDB_GET_INT32(pRaw, dataPos, &arbGroupId, _OVER) + taosHashPut(pTrans->arbGroupIds, &arbGroupId, sizeof(int32_t), NULL, 0); + } + SDB_GET_RESERVE(pRaw, dataPos, TRANS_RESERVE_SIZE, _OVER) terrno = 0; @@ -462,6 +484,9 @@ void mndTransDropData(STrans *pTrans) { mndTransDropActions(pTrans->commitActions); pTrans->commitActions = NULL; } + if (pTrans->arbGroupIds != NULL) { + taosHashCleanup(pTrans->arbGroupIds); + } if (pTrans->pRpcArray != NULL) { taosArrayDestroy(pTrans->pRpcArray); pTrans->pRpcArray = NULL; @@ -581,6 +606,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, pTrans->redoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->undoActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); pTrans->commitActions = taosArrayInit(TRANS_ARRAY_SIZE, sizeof(STransAction)); + pTrans->arbGroupIds = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); pTrans->pRpcArray = taosArrayInit(1, sizeof(SRpcHandleInfo)); pTrans->mTraceId = pReq ? TRACE_GET_ROOTID(&pReq->info.traceId) : tGenIdPI64(); taosInitRWLatch(&pTrans->lockRpcArray); @@ -733,7 +759,9 @@ void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) } } -void mndTransSetArbGroupId(STrans *pTrans, int32_t groupId) { pTrans->arbGroupId = groupId; } +void mndTransAddArbGroupId(STrans *pTrans, int32_t groupId) { + taosHashPut(pTrans->arbGroupIds, &groupId, sizeof(int32_t), NULL, 0); +} void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } @@ -768,7 +796,13 @@ static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { static bool mndCheckDbConflict(const char *conflict, STrans *pTrans) { if (conflict[0] == 0) return false; - if (strcasecmp(conflict, pTrans->dbname) == 0 || strcasecmp(conflict, pTrans->stbname) == 0) return true; + if (strcasecmp(conflict, pTrans->dbname) == 0) return true; + return false; +} + +static bool mndCheckStbConflict(const char *conflict, STrans *pTrans) { + if (conflict[0] == 0) return false; + if (strcasecmp(conflict, pTrans->stbname) == 0) return true; return false; } @@ -788,17 +822,17 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (mndCheckDbConflict(pNew->dbname, pTrans)) conflict = true; - if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; + if (mndCheckStbConflict(pNew->stbname, pTrans)) conflict = true; } } if (pNew->conflict == TRN_CONFLICT_DB_INSIDE) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_DB) { if (mndCheckDbConflict(pNew->dbname, pTrans)) conflict = true; - if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; + if (mndCheckStbConflict(pNew->stbname, pTrans)) conflict = true; } if (pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { - if (mndCheckDbConflict(pNew->stbname, pTrans)) conflict = true; // for stb + if (mndCheckStbConflict(pNew->stbname, pTrans)) conflict = true; // for stb } } @@ -821,7 +855,16 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (pNew->conflict == TRN_CONFLICT_ARBGROUP) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) { - if (pNew->arbGroupId == pTrans->arbGroupId) conflict = true; + void *pIter = taosHashIterate(pNew->arbGroupIds, NULL); + while (pIter != NULL) { + int32_t groupId = *(int32_t *)pIter; + if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) { + taosHashCancelIterate(pNew->arbGroupIds, pIter); + conflict = true; + break; + } + pIter = taosHashIterate(pNew->arbGroupIds, pIter); + } } } @@ -1372,7 +1415,7 @@ static int32_t mndTransExecuteActionsSerial(SMnode *pMnode, STrans *pTrans, SArr mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->actionPos); for (int32_t action = pTrans->actionPos; action < numOfActions; ++action) { - STransAction *pAction = taosArrayGet(pActions, pTrans->actionPos); + STransAction *pAction = taosArrayGet(pActions, action); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 64bca96702..ee60556639 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -49,6 +49,7 @@ TEST_F(MndTestFunc, 01_Show_Func) { } TEST_F(MndTestFunc, 02_Create_Func) { +#ifndef WINDOWS { SCreateFuncReq createReq = {0}; strcpy(createReq.name, ""); @@ -159,9 +160,11 @@ TEST_F(MndTestFunc, 02_Create_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); +#endif } TEST_F(MndTestFunc, 03_Retrieve_Func) { +#ifndef WINDOWS { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; @@ -376,9 +379,11 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } +#endif } TEST_F(MndTestFunc, 04_Drop_Func) { +#ifndef WINDOWS { SDropFuncReq dropReq = {0}; strcpy(dropReq.name, ""); @@ -441,9 +446,11 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "ins_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); +#endif } TEST_F(MndTestFunc, 05_Actual_code) { +#ifndef WINDOWS { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "udf1"); @@ -507,4 +514,5 @@ TEST_F(MndTestFunc, 05_Actual_code) { } tFreeSRetrieveFuncRsp(&retrieveRsp); } -} \ No newline at end of file +#endif +} diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 8480f204d6..c9365b4318 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -138,7 +138,6 @@ void initStreamExecInfo() { } void initNodeInfo() { - execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); SNodeEntry entry = {0}; entry.nodeId = 2; entry.stageUpdated = true; @@ -207,27 +206,32 @@ TEST_F(StreamTest, kill_checkpoint_trans) { killAllCheckpointTrans(pMnode, &info); - SStreamObj stream; - memset(&stream, 0, sizeof(SStreamObj)); + void* p = alloca(sizeof(SStreamObj) + sizeof(SSdbRow)); + SSdbRow* pRow = static_cast(p); + pRow->type = SDB_MAX; - stream.uid = defStreamId; - stream.lock = 0; - stream.tasks = taosArrayInit(1, POINTER_BYTES); - stream.pHTasksList = taosArrayInit(1, POINTER_BYTES); + SStreamObj* pStream = (SStreamObj*)((char*)p + sizeof(SSdbRow)); + + memset(pStream, 0, sizeof(SStreamObj)); + + pStream->uid = defStreamId; + pStream->lock = 0; + pStream->tasks = taosArrayInit(1, POINTER_BYTES); + pStream->pHTasksList = taosArrayInit(1, POINTER_BYTES); SArray* pLevel = taosArrayInit(1, POINTER_BYTES); SStreamTask* pTask = static_cast(taosMemoryCalloc(1, sizeof(SStreamTask))); pTask->id.streamId = defStreamId; pTask->id.taskId = 1; - pTask->exec.qmsg = (char*)taosMemoryMalloc(1); + pTask->exec.qmsg = (char*)taosMemoryCalloc(1,1); taosThreadMutexInit(&pTask->lock, NULL); taosArrayPush(pLevel, &pTask); - taosArrayPush(stream.tasks, &pLevel); - mndCreateStreamResetStatusTrans(pMnode, &stream); + taosArrayPush(pStream->tasks, &pLevel); + mndCreateStreamResetStatusTrans(pMnode, pStream); - tFreeStreamObj(&stream); + tFreeStreamObj(pStream); sdbCleanup(pMnode->pSdb); taosMemoryFree(pMnode); diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 8c5d056893..2ac66fa1cd 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -35,23 +35,6 @@ struct SSnode { SMsgCb msgCb; }; -#if 0 -typedef struct { - SHashObj* pHash; // taskId -> SStreamTask -} SStreamMeta; - -SStreamMeta* sndMetaNew(); -void sndMetaDelete(SStreamMeta* pMeta); - -int32_t sndMetaDeployTask(SStreamMeta* pMeta, SStreamTask* pTask); -SStreamTask* sndMetaGetTask(SStreamMeta* pMeta, int32_t taskId); -int32_t sndMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); - -int32_t sndDropTaskOfStream(SStreamMeta* pMeta, int64_t streamId); -int32_t sndStopTaskOfStream(SStreamMeta* pMeta, int64_t streamId); -int32_t sndResumeTaskOfStream(SStreamMeta* pMeta, int64_t streamId); -#endif - void initStreamStateAPI(SStorageAPI* pAPI); #ifdef __cplusplus diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 87f0681780..c61988574c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -45,15 +45,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer if (code != TSDB_CODE_SUCCESS) { return code; } + pTask->pBackend = NULL; - streamTaskOpenAllUpstreamInput(pTask); - code = tqExpandStreamTask(pTask, pSnode->pMeta, NULL); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); @@ -67,14 +62,14 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); + (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam); } else { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, - (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); + (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam); } return 0; } @@ -86,6 +81,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { return NULL; } + stopRsync(); + startRsync(); + pSnode->msgCb = pOption->msgCb; pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); if (pSnode->pMeta == NULL) { @@ -94,10 +92,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { } streamMetaLoadAllTasks(pSnode->pMeta); - - stopRsync(); - startRsync(); - return pSnode; FAIL: @@ -106,8 +100,7 @@ FAIL: } int32_t sndInit(SSnode *pSnode) { - streamMetaResetTaskStatus(pSnode->pMeta); - streamMetaStartAllTasks(pSnode->pMeta); + streamTaskSchedTask(&pSnode->msgCb, pSnode->pMeta->vgId, 0, 0, STREAM_EXEC_T_START_ALL_TASKS); return 0; } @@ -143,6 +136,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg); + case TDMT_STREAM_RETRIEVE_TRIGGER: + return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg); + case TDMT_STREAM_RETRIEVE_TRIGGER_RSP: + return tqStreamTaskProcessRetrieveTriggerRsp(pSnode->pMeta, pMsg); default: sndError("invalid snode msg:%d", pMsg->msgType); ASSERT(0); @@ -168,6 +165,8 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return tqStreamTaskProcessTaskPauseReq(pSnode->pMeta, pMsg->pCont); case TDMT_STREAM_TASK_RESUME: return tqStreamTaskProcessTaskResumeReq(pSnode->pMeta, pMsg->info.conn.applyIndex, pMsg->pCont, false); + case TDMT_STREAM_TASK_UPDATE_CHKPT: + return tqStreamTaskProcessUpdateCheckpointReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); default: ASSERT(0); } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 3b60ef3427..196fa56c99 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -75,8 +75,10 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateCountGetKeyByRange = streamStateCountGetKeyByRange; pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition; - pStore->streamStateCountWinAddIfNotExist = streamStateCountWinAddIfNotExist; - pStore->streamStateCountWinAdd = streamStateCountWinAdd; +//void initStreamStateAPI(SStorageAPI* pAPI) { +// initStateStoreAPI(&pAPI->stateStore); +// initFunctionStateStore(&pAPI->functionStore); +//} pStore->updateInfoInit = updateInfoInit; pStore->updateInfoFillBlockData = updateInfoFillBlockData; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e2ecdca59f..651fe2cda4 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -113,7 +113,7 @@ int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle); void tqDestroyTqHandle(void* data); // tqRead -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset); +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* offset); int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest); int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); @@ -160,7 +160,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const void* pRsp, int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, - SSubmitTbData* pTableData, const char* id); + SSubmitTbData* pTableData, int64_t earlyTs, const char* id); int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id); SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols, diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c85316f810..84e1996f2c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -355,6 +355,8 @@ typedef struct { int flush_count; } SCacheFlushState; +typedef struct SCompMonitor SCompMonitor; + struct STsdb { char * path; SVnode * pVnode; @@ -375,8 +377,11 @@ struct STsdb { TdThreadMutex pgMutex; struct STFileSystem *pFS; // new SRocksCache rCache; - // compact monitor - struct SCompMonitor *pCompMonitor; + SCompMonitor *pCompMonitor; + struct { + SVHashTable *ht; + SArray *arr; + } *commitInfo; }; struct TSDBKEY { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 304716744c..7bdc8e1c33 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -49,31 +49,21 @@ int32_t vnodeEncodeConfig(const void* pObj, SJson* pJson); int32_t vnodeDecodeConfig(const SJson* pJson, void* pObj); // vnodeAsync.c -typedef struct SVAsync SVAsync; - typedef enum { EVA_PRIORITY_HIGH = 0, EVA_PRIORITY_NORMAL, EVA_PRIORITY_LOW, } EVAPriority; -#define VNODE_ASYNC_VALID_CHANNEL_ID(channelId) ((channelId) > 0) -#define VNODE_ASYNC_VALID_TASK_ID(taskId) ((taskId) > 0) - -int32_t vnodeAsyncInit(SVAsync** async, char* label); -int32_t vnodeAsyncDestroy(SVAsync** async); -int32_t vnodeAChannelInit(SVAsync* async, int64_t* channelId); -int32_t vnodeAChannelDestroy(SVAsync* async, int64_t channelId, bool waitRunning); -int32_t vnodeAsync(SVAsync* async, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), void* arg, - int64_t* taskId); -int32_t vnodeAsyncC(SVAsync* async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void*), - void (*complete)(void*), void* arg, int64_t* taskId); -int32_t vnodeAWait(SVAsync* async, int64_t taskId); -int32_t vnodeACancel(SVAsync* async, int64_t taskId); -int32_t vnodeAsyncSetWorkers(SVAsync* async, int32_t numWorkers); - -// vnodeModule.c -extern SVAsync* vnodeAsyncHandle[2]; +int32_t vnodeAsyncOpen(int32_t numOfThreads); +int32_t vnodeAsyncClose(); +int32_t vnodeAChannelInit(int64_t async, SVAChannelID* channelID); +int32_t vnodeAChannelDestroy(SVAChannelID* channelID, bool waitRunning); +int32_t vnodeAsync(SVAChannelID* channelID, EVAPriority priority, int32_t (*execute)(void*), void (*complete)(void*), + void* arg, SVATaskID* taskID); +int32_t vnodeAWait(SVATaskID* taskID); +int32_t vnodeACancel(SVATaskID* taskID); +int32_t vnodeAsyncSetWorkers(int64_t async, int32_t numWorkers); // vnodeBufPool.c typedef struct SVBufPoolNode SVBufPoolNode; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 419ebd1a6c..8222af4d60 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -237,14 +237,12 @@ int32_t tsdbCacheNewSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, int8_t int32_t tsdbCacheDropSTableColumn(STsdb* pTsdb, SArray* uids, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t col_type); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); -int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); -int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); -int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); +int64_t tsdbGetEarliestTs(STsdb* pTsdb); // tq STQ* tqOpen(const char* path, SVnode* pVnode); @@ -257,6 +255,8 @@ int tqScanWalAsync(STQ* pTq, bool ckPause); int32_t tqStopStreamTasksAsync(STQ* pTq); int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp); int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRetrieveTriggerRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); @@ -295,6 +295,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg); int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen); // sma int32_t smaInit(); @@ -470,6 +471,16 @@ typedef struct SVMonitorObj { taos_counter_t* insertCounter; } SVMonitorObj; +typedef struct { + int64_t async; + int64_t id; +} SVAChannelID; + +typedef struct { + int64_t async; + int64_t id; +} SVATaskID; + struct SVnode { char* path; SVnodeCfg config; @@ -491,8 +502,8 @@ struct SVnode { SVBufPool* onRecycle; // commit variables - int64_t commitChannel; - int64_t commitTask; + SVAChannelID commitChannel; + SVATaskID commitTask; SMeta* pMeta; SSma* pSma; @@ -598,6 +609,24 @@ struct SCompactInfo { void initStorageAPI(SStorageAPI* pAPI); +// a simple hash table impl +typedef struct SVHashTable SVHashTable; + +struct SVHashTable { + uint32_t (*hash)(const void*); + int32_t (*compare)(const void*, const void*); + int32_t numEntries; + uint32_t numBuckets; + struct SVHashEntry** buckets; +}; + +#define vHashNumEntries(ht) ((ht)->numEntries) +int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); +int32_t vHashDestroy(SVHashTable** ht); +int32_t vHashPut(SVHashTable* ht, void* obj); +int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj); +int32_t vHashDrop(SVHashTable* ht, const void* obj); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index df54591ae8..2a122081b8 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -36,7 +36,7 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); -int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) { +int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add, uint32_t compress) { int32_t nCols = pWp->nCols; int32_t ver = pWp->version; if (add) { @@ -45,7 +45,7 @@ int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) { SColCmpr *pCol = p + nCols; pCol->id = pSchema->colId; - pCol->alg = createDefaultColCmprByType(pSchema->type); + pCol->alg = compress; pWp->nCols = nCols + 1; pWp->version = ver; pWp->pColCmpr = p; @@ -1491,6 +1491,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl SSchema tScheam; switch (pAlterTbReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: if (pColumn) { terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; goto _err; @@ -1522,7 +1523,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl (void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); } SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1]; - updataTableColCmpr(&entry.colCmpr, pCol, 1); + uint32_t compress = pAlterTbReq->action == TSDB_ALTER_TABLE_ADD_COLUMN ? createDefaultColCmprByType(pCol->type) + : pAlterTbReq->compress; + updataTableColCmpr(&entry.colCmpr, pCol, 1, compress); freeColCmpr = true; ASSERT(entry.colCmpr.nCols == pSchema->nCols); break; @@ -1560,7 +1563,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl (void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey); } - updataTableColCmpr(&entry.colCmpr, &tScheam, 0); + updataTableColCmpr(&entry.colCmpr, &tScheam, 0, 0); ASSERT(entry.colCmpr.nCols == pSchema->nCols); break; case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: @@ -2269,6 +2272,7 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta pMeta->changed = true; switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 5441d0c4c1..3cc7c6ec66 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -244,7 +244,7 @@ static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTask SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask && *ppTask) { pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer; - pItem->fetchResultVer = (*ppTask)->info.triggerParam; + pItem->fetchResultVer = (*ppTask)->info.delaySchedParam; } streamMetaRUnLock(pMeta); } @@ -298,7 +298,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); - + pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; @@ -622,8 +622,8 @@ int32_t smaRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1); - if (code) goto _end; + // code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1); + // if (code) goto _end; } } @@ -1285,10 +1285,11 @@ _checkpoint: if (pItem && pItem->pStreamTask) { SStreamTask *pTask = pItem->pStreamTask; // atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1); - pTask->chkInfo.checkpointingId = checkpointId; + streamTaskSetActiveCheckpointInfo(pTask, checkpointId); + pTask->chkInfo.checkpointId = checkpointId; // 1pTask->checkpointingId; pTask->chkInfo.checkpointVer = pItem->submitReqVer; - pTask->info.triggerParam = pItem->fetchResultVer; + pTask->info.delaySchedParam = pItem->fetchResultVer; pTask->info.taskLevel = TASK_LEVEL_SMA; if (!checkpointBuilt) { diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index fb898c02f8..d0913081ac 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -203,7 +203,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId)); if (index == NULL) { // no data yet, append it - code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, INT64_MIN, ""); if (code != TSDB_CODE_SUCCESS) { continue; } @@ -213,7 +213,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1; taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); } else { - code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, ""); + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, INT64_MIN, ""); if (code != TSDB_CODE_SUCCESS) { continue; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 79f53e6dec..712cfbaa55 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -717,17 +717,14 @@ static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { int32_t vgId = TD_VID(pTq->pVnode); - tqDebug("s-task:0x%x start to expand task", pTask->id.taskId); + tqDebug("s-task:0x%x start to build task", pTask->id.taskId); int32_t code = streamTaskInit(pTask, pTq->pStreamMeta, &pTq->pVnode->msgCb, nextProcessVer); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tqExpandStreamTask(pTask, pTq->pStreamMeta, pTq->pVnode); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + pTask->pBackend = NULL; // sink STaskOutputInfo* pOutputInfo = &pTask->outputInfo; @@ -770,22 +767,22 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { const char* pNext = streamTaskGetStatusStr(pTask->status.taskStatus); if (pTask->info.fillHistory) { - tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + tqInfo("vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, cur-status:%s, next-status:%s fill-history:%d, related stream task:0x%x " - "trigger:%" PRId64 " ms, inputVer:%" PRId64, + "delaySched:%" PRId64 " ms, inputVer:%" PRId64, vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory, - (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam, nextProcessVer); + (int32_t)pTask->streamTaskId.taskId, pTask->info.delaySchedParam, nextProcessVer); } else { tqInfo( - "vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + "vgId:%d build stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 - " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 + " child id:%d, level:%d, cur-status:%s next-status:%s fill-history:%d, related fill-task:0x%x delaySched:%" PRId64 " ms, inputVer:%" PRId64, vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pNext, pTask->info.fillHistory, - (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam, nextProcessVer); + (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.delaySchedParam, nextProcessVer); ASSERT(pChkInfo->checkpointVer <= pChkInfo->nextProcessVer); } @@ -800,6 +797,11 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { +// if (!pTq->pVnode->restored) { +// tqDebug("vgId:%d not restored, ignore the stream task deploy msg", TD_VID(pTq->pVnode)); +// return TSDB_CODE_SUCCESS; +// } + return tqStreamTaskProcessDeployReq(pTq->pStreamMeta, &pTq->pVnode->msgCb, sversion, msg, msgLen, vnodeIsRoleLeader(pTq->pVnode), pTq->pVnode->restored); } @@ -828,7 +830,7 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask streamSetParamForStreamScannerStep2(pTask, pStep2Range, pWindow); - int64_t dstVer =pStep2Range->minVer; + int64_t dstVer = pStep2Range->minVer; pTask->chkInfo.nextProcessVer = dstVer; walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); @@ -1009,6 +1011,20 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { return tqStreamTaskProcessDropReq(pTq->pStreamMeta, msg, msgLen); } +int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) { + int32_t vgId = TD_VID(pTq->pVnode); + SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; + +// if (!pTq->pVnode->restored) { +// tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64 +// ", transId:%d s-task:0x%x ignore it", +// vgId, pReq->checkpointId, pReq->transId, pReq->taskId); +// return TSDB_CODE_SUCCESS; +// } + + return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen); +} + int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { return tqStreamTaskProcessTaskPauseReq(pTq->pStreamMeta, msg); } @@ -1120,10 +1136,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) } if (pTask->status.downstreamReady != 1) { - pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id - pTask->chkInfo.checkpointingId = req.checkpointId; - pTask->chkInfo.transId = req.transId; - + streamTaskSetFailedChkptInfo(pTask, req.transId, req.checkpointId); // record the latest failed checkpoint id tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpointId:%" PRId64 ", transId:%d set it failed", pTask->id.idStr, req.checkpointId, req.transId); @@ -1162,9 +1175,12 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { + int64_t checkpointId = 0; + streamTaskGetActiveCheckpointInfo(pTask, NULL, &checkpointId); + tqWarn("s-task:%s repeatly recv checkpoint-source msg checkpointId:%" PRId64 " transId:%d already handled, ignore msg and continue process checkpoint", - pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); + pTask->id.idStr, checkpointId, req.transId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); @@ -1213,6 +1229,15 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg) { + int32_t vgId = TD_VID(pTq->pVnode); + + SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont; + if (!vnodeIsRoleLeader(pTq->pVnode)) { + tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId, + (int32_t)pReq->downstreamTaskId); + return TSDB_CODE_STREAM_NOT_LEADER; + } + return tqStreamTaskProcessCheckpointReadyMsg(pTq->pStreamMeta, pMsg); } @@ -1224,6 +1249,23 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } +int32_t tqProcessTaskRetrieveTriggerReq(STQ* pTq, SRpcMsg* pMsg) { + int32_t vgId = TD_VID(pTq->pVnode); + + SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont; + if (!vnodeIsRoleLeader(pTq->pVnode)) { + tqError("vgId:%d not leader, ignore the retrieve checkpoint-trigger msg from 0x%x", vgId, + (int32_t)pReq->downstreamTaskId); + return TSDB_CODE_STREAM_NOT_LEADER; + } + + return tqStreamTaskProcessRetrieveTriggerReq(pTq->pStreamMeta, pMsg); +} + +int32_t tqProcessTaskRetrieveTriggerRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamTaskProcessRetrieveTriggerRsp(pTq->pStreamMeta, pMsg); +} + // this function is needed, do not try to remove it. int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg) { return tqStreamProcessStreamHbRsp(pTq->pStreamMeta, pMsg); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 76322c527f..404cbf26dd 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -178,6 +178,7 @@ int32_t tqMetaRestoreCheckInfo(STQ* pTq) { goto END; } } + END: tdbFree(pKey); tdbFree(pVal); @@ -514,35 +515,6 @@ int32_t tqMetaTransform(STQ* pTq) { return code; } -//int32_t tqMetaRestoreHandle(STQ* pTq) { -// int code = 0; -// TBC* pCur = NULL; -// if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { -// return -1; -// } -// -// void* pKey = NULL; -// int kLen = 0; -// void* pVal = NULL; -// int vLen = 0; -// -// tdbTbcMoveToFirst(pCur); -// -// while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { -// STqHandle handle = {0}; -// code = restoreHandle(pTq, pVal, vLen, &handle); -// if (code < 0) { -// tqDestroyTqHandle(&handle); -// break; -// } -// } -// -// tdbFree(pKey); -// tdbFree(pVal); -// tdbTbcClose(pCur); -// return code; -//} - int32_t tqMetaGetHandle(STQ* pTq, const char* key) { void* pVal = NULL; int vLen = 0; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 516a47606b..d3be8fa666 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -99,14 +99,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { taosArrayDestroy(reqNew.pArray); } - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - taosMemoryFreeClear(pCreateReq->comment); - taosMemoryFreeClear(pCreateReq->sql); - if (pCreateReq->type == TSDB_CHILD_TABLE) { - taosArrayDestroy(pCreateReq->ctb.tagName); - } - } + tDeleteSVCreateTbBatchReq(&req); } else if (msgType == TDMT_VND_ALTER_TABLE) { SVAlterTbReq req = {0}; @@ -385,7 +378,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { SWalReader* pWalReader = pReader->pWalReader; - uint64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampMs(); while (1) { int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { @@ -413,7 +406,8 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->msg.msgStr = NULL; - if (taosGetTimestampMs() - st > 1000) { + int64_t elapsed = taosGetTimestampMs() - st; + if(elapsed > 1000 || elapsed < 0){ return false; } @@ -1089,7 +1083,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + if ((pTask->info.taskLevel == TASK_LEVEL__SOURCE) && (pTask->exec.pExecutor != NULL)) { int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 08f1689f2f..b9c9595c74 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -163,7 +163,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* return 0; } -int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { +int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqBatchMetaRsp* pBatchMetaRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; qTaskInfo_t task = pExec->task; @@ -218,10 +218,10 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } // get meta - SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); - if (tmp->metaRspLen > 0) { + SMqBatchMetaRsp* tmp = qStreamExtractMetaMsg(task); + if (taosArrayGetSize(tmp->batchMetaReq) > 0) { qStreamExtractOffset(task, &tmp->rspOffset); - *pMetaRsp = *tmp; + *pBatchMetaRsp = *tmp; tqDebug("tmqsnap task get meta"); break; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index c0f58fc3ec..5f3e1e3d14 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -34,11 +34,12 @@ static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSData static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks); static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, - const char* id); + int64_t earlyTs, const char* id); static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo, const char* dstTableName, int64_t* uid); static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id); +static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags); @@ -71,7 +72,7 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p if (varTbName != NULL && varTbName != (void*)-1) { name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); memcpy(name, varDataVal(varTbName), varDataLen(varTbName)); - if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name) && groupId != 0 && stbFullName) { + if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) { buildCtbNameAddGroupId(stbFullName, name, groupId); } } else if (stbFullName) { @@ -182,7 +183,7 @@ void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDa int64_t gid, bool newSubTableRule) { if (pDataBlock->info.parTbName[0]) { if (newSubTableRule && !isAutoTableName(pDataBlock->info.parTbName) && - !alreadyAddGroupId(pDataBlock->info.parTbName) && gid != 0 && stbFullName) { + !alreadyAddGroupId(pDataBlock->info.parTbName, gid) && gid != 0 && stbFullName) { pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); strcpy(pCreateTableReq->name, pDataBlock->info.parTbName); buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid); @@ -396,46 +397,6 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c return TSDB_CODE_SUCCESS; } -int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, - int64_t suid) { - SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; - - int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr, - pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { - taosArrayDestroy(deleteReq.deleteReqs); - return TSDB_CODE_SUCCESS; - } - - int32_t len; - tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); - if (code != TSDB_CODE_SUCCESS) { - qError("s-task:%s failed to encode delete request", pTask->id.idStr); - return code; - } - - SEncoder encoder; - void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); - void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); - tEncoderClear(&encoder); - taosArrayDestroy(deleteReq.deleteReqs); - - ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); - - SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - tqDebug("failed to put delete req into write-queue since %s", terrstr()); - } - - return TSDB_CODE_SUCCESS; -} - bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid) { if (pReader->me.type != TSDB_CHILD_TABLE) { tqError("vgId:%d, failed to write into %s, since table type:%d incorrect", vgId, ctbName, pReader->me.type); @@ -484,23 +445,6 @@ SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, in return pCreateTbReq; } -int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { - if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) { - taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it - return TSDB_CODE_FAILED; - } - - int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pTableSinkInfo); - } else { - tqDebug("s-task:%s new dst table:%s(uid:%" PRIu64 ") added into cache, total:%d", id, pTableSinkInfo->name.data, - pTableSinkInfo->uid, tSimpleHashGetSize(pSinkTableMap)); - } - - return code; -} - int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen) { int32_t code = 0; void* pBuf = NULL; @@ -552,7 +496,8 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) { } } -int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) { +int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, int64_t earlyTs, + const char* id) { int32_t numOfRows = pDataBlock->info.rows; int32_t code = TSDB_CODE_SUCCESS; @@ -581,6 +526,14 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); ts = *(int64_t*)colDataGetData(pColData, j); tqTrace("s-task:%s sink row %d, col %d ts %" PRId64, id, j, k, ts); + + if (ts < earlyTs) { + tqError("s-task:%s ts:%" PRId64 " of generated results out of valid time range %" PRId64 " , discarded", id, + ts, earlyTs); + pTableData->aRowP = taosArrayDestroy(pTableData->aRowP); + taosArrayDestroy(pVals); + return TSDB_CODE_SUCCESS; + } } if (IS_SET_NULL(pCol)) { @@ -605,8 +558,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat dataIndex++; } else { void* colData = colDataGetData(pColData, j); - if (IS_VAR_DATA_TYPE(pCol->type)) { - // address copy, no value + if (IS_VAR_DATA_TYPE(pCol->type)) { // address copy, no value SValue sv = (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; SColVal cv = COL_VAL_VALUE(pCol->colId, sv); @@ -713,7 +665,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat buildCtbNameByGroupIdImpl(stbFullName, groupId, dstTableName); } else { if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(dstTableName) && - !alreadyAddGroupId(dstTableName) && groupId != 0) { + !alreadyAddGroupId(dstTableName, groupId) && groupId != 0) { tqDebug("s-task:%s append groupId:%" PRId64 " for generated dstTable:%s", id, groupId, dstTableName); if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ buildCtbNameAddGroupId(NULL, dstTableName, groupId); @@ -744,7 +696,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat tqTrace("s-task:%s set the dstTable uid from cache:%" PRId64, id, pTableData->uid); } } else { - // The auto-create option will always set to be open for those submit messages, which arrive during the period + // The auto-create option will always set to be open for those submit messages, which arrives during the period // the creating of the destination table, due to the absence of the user-specified table in TSDB. When scanning // data from WAL, those submit messages, with auto-created table option, will be discarded expect the first, for // those mismatched table uids. Only the FIRST table has the correct table uid, and those remain all have @@ -752,7 +704,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat SMetaReader mr = {0}; metaReaderDoInit(&mr, pVnode->pMeta, META_READER_LOCK); - // table not in cache, let's try the extract it from tsdb meta + // table not in cache, let's try to extract it from tsdb meta if (metaGetTableEntryByName(&mr, dstTableName) < 0) { metaReaderClear(&mr); @@ -796,34 +748,26 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat } int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, - SSubmitTbData* pTableData, const char* id) { + SSubmitTbData* pTableData, int64_t earlyTs, const char* id) { int32_t numOfRows = pDataBlock->info.rows; + char* dstTableName = pDataBlock->info.parTbName; tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id, blockIndex + 1, numOfRows, suid); - char* dstTableName = pDataBlock->info.parTbName; // convert all rows - int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id); + int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, earlyTs, id); if (code != TSDB_CODE_SUCCESS) { tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno)); return code; } - taosArraySort(pTableData->aRowP, tsAscendingSortFn); - tqTrace("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); - return code; -} - -bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) { - for (int32_t i = 0; i < numOfBlocks; ++i) { - SSDataBlock* p = taosArrayGet(pBlocks, i); - if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { - return false; - } + if (pTableData->aRowP != NULL) { + taosArraySort(pTableData->aRowP, tsAscendingSortFn); + tqTrace("s-task:%s build submit msg for dstTable:%s, numOfRows:%d", id, dstTableName, numOfRows); } - return true; + return code; } void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { @@ -836,6 +780,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { int32_t numOfBlocks = taosArrayGetSize(pBlocks); int32_t code = TSDB_CODE_SUCCESS; const char* id = pTask->id.idStr; + int64_t earlyTs = tsdbGetEarliestTs(pVnode->pTsdb); bool onlySubmitData = hasOnlySubmitData(pBlocks, numOfBlocks); if (!onlySubmitData) { @@ -870,8 +815,13 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { continue; } - code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); - if (code != TSDB_CODE_SUCCESS) { + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); + if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { + if (tbData.pCreateTbReq != NULL) { + tdDestroySVCreateTbReq(tbData.pCreateTbReq); + doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, pDataBlock->info.id.groupId, id); + tbData.pCreateTbReq = NULL; + } continue; } @@ -918,8 +868,13 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { continue; } - code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); - if (code != TSDB_CODE_SUCCESS) { + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); + if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { + if (tbData.pCreateTbReq != NULL) { + tdDestroySVCreateTbReq(tbData.pCreateTbReq); + doRemoveFromCache(pTask->outputInfo.tbSink.pTblInfo, groupId, id); + tbData.pCreateTbReq = NULL; + } continue; } @@ -928,8 +883,12 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); } else { - code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id); - if (code != TSDB_CODE_SUCCESS) { + code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, earlyTs, id); + if (code != TSDB_CODE_SUCCESS || tbData.aRowP == NULL) { + if (tbData.pCreateTbReq != NULL) { + tdDestroySVCreateTbReq(tbData.pCreateTbReq); + tbData.pCreateTbReq = NULL; + } continue; } @@ -953,3 +912,82 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) { } } } + +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks) { + for (int32_t i = 0; i < numOfBlocks; ++i) { + SSDataBlock* p = taosArrayGet(pBlocks, i); + if (p->info.type == STREAM_DELETE_RESULT || p->info.type == STREAM_CREATE_CHILD_TABLE) { + return false; + } + } + + return true; +} + +int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id) { + if (tSimpleHashGetSize(pSinkTableMap) > MAX_CACHE_TABLE_INFO_NUM) { + taosMemoryFreeClear(pTableSinkInfo); // too many items, failed to cache it + return TSDB_CODE_FAILED; + } + + int32_t code = tSimpleHashPut(pSinkTableMap, &groupId, sizeof(uint64_t), &pTableSinkInfo, POINTER_BYTES); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pTableSinkInfo); + } else { + tqDebug("s-task:%s new dst table:%s(uid:%" PRIu64 ") added into cache, total:%d", id, pTableSinkInfo->name.data, + pTableSinkInfo->uid, tSimpleHashGetSize(pSinkTableMap)); + } + + return code; +} + +int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char* id) { + if (tSimpleHashGetSize(pSinkTableMap) == 0) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = tSimpleHashRemove(pSinkTableMap, &groupId, sizeof(groupId)); + tqDebug("s-task:%s remove cached table meta for groupId:%" PRId64, id, groupId); + return code; +} + +int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, + int64_t suid) { + SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; + + int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr, + pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + return TSDB_CODE_SUCCESS; + } + + int32_t len; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code != TSDB_CODE_SUCCESS) { + qError("s-task:%s failed to encode delete request", pTask->id.idStr); + return code; + } + + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = TD_VID(pVnode); + + SRpcMsg msg = {.msgType = TDMT_VND_BATCH_DEL, .pCont = serializedDeleteReq, .contLen = len + sizeof(SMsgHead)}; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 5a29f67ae3..bcf17bf1e1 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -291,8 +291,7 @@ bool doPutDataIntoInputQ(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems } } else { walReaderSeekVer(pTask->exec.pWalReader, pTask->chkInfo.nextProcessVer); - tqError("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, - pTask->chkInfo.nextProcessVer); + tqTrace("s-task:%s append input queue failed, code:too many items, ver:%" PRId64, id, pTask->chkInfo.nextProcessVer); break; } } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 28f4a19949..33e3414a7d 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -17,6 +17,8 @@ static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); +static int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, + const SMqBatchMetaRsp* pRsp, int32_t vgId); int32_t tqInitDataRsp(SMqDataRspCommon* pRsp, STqOffsetVal pOffset) { tOffsetCopy(&pRsp->reqOffset, &pOffset); @@ -172,7 +174,7 @@ end : { } } -#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC) \ +#define PROCESS_EXCLUDED_MSG(TYPE, DECODE_FUNC, DELETE_FUNC) \ SDecoder decoder = {0};\ TYPE req = {0}; \ void* data = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)); \ @@ -182,33 +184,37 @@ end : { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, jump meta for, vgId:%d offset %" PRId64 " msgType %d", \ pRequest->consumerId, pRequest->epoch, vgId, fetchVer, pHead->msgType); \ fetchVer++; \ + DELETE_FUNC(&req); \ tDecoderClear(&decoder); \ continue; \ } \ + DELETE_FUNC(&req); \ tDecoderClear(&decoder); +static void tDeleteCommon(void* parm) { +} static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { - int code = 0; - int32_t vgId = TD_VID(pTq->pVnode); - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + STaosxRsp taosxRsp = {0}; + SMqBatchMetaRsp btMetaRsp = {0}; tqInitTaosxRsp(&taosxRsp.common, *offset); if (offset->type != TMQ_OFFSET__LOG) { - if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &btMetaRsp, offset) < 0) { code = -1; goto end; } - if (metaRsp.metaRspLen > 0) { - code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); + if (taosArrayGetSize(btMetaRsp.batchMetaReq) > 0) { + code = tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, - pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, - metaRsp.rspOffset.ts); - tDeleteMqMetaRsp(&metaRsp); + pRequest->consumerId, pHandle->subKey, vgId, btMetaRsp.rspOffset.type, btMetaRsp.rspOffset.uid, + btMetaRsp.rspOffset.ts); + tDeleteMqBatchMetaRsp(&btMetaRsp); goto end; } @@ -230,11 +236,18 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, uint64_t st = taosGetTimestampMs(); int totalRows = 0; + int32_t totalMetaRows = 0; while (1) { int32_t savedEpoch = atomic_load_32(&pHandle->epoch); ASSERT(savedEpoch <= pRequest->epoch); if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { + if (totalMetaRows > 0) { + tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); + tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); + ASSERT(totalRows == 0); + goto end; + } tqOffsetResetToLog(&taosxRsp.common.rspOffset, fetchVer); code = tqSendDataRsp( pHandle, pMsg, pRequest, &taosxRsp, @@ -258,22 +271,69 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if ((pRequest->sourceExcluded & TD_REQ_FROM_TAOX) != 0) { if (pHead->msgType == TDMT_VND_CREATE_TABLE) { - PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq) + PROCESS_EXCLUDED_MSG(SVCreateTbBatchReq, tDecodeSVCreateTbBatchReq, tDeleteSVCreateTbBatchReq) } else if (pHead->msgType == TDMT_VND_ALTER_TABLE) { - PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq) + PROCESS_EXCLUDED_MSG(SVAlterTbReq, tDecodeSVAlterTbReq, tDeleteCommon) } else if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB) { - PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq) + PROCESS_EXCLUDED_MSG(SVCreateStbReq, tDecodeSVCreateStbReq, tDeleteCommon) } else if (pHead->msgType == TDMT_VND_DELETE) { - PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes) + PROCESS_EXCLUDED_MSG(SDeleteRes, tDecodeDeleteRes, tDeleteCommon) } } - tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); - metaRsp.resMsgType = pHead->msgType; - metaRsp.metaRspLen = pHead->bodyLen; - metaRsp.metaRsp = pHead->body; - code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); + tqDebug("fetch meta msg, ver:%" PRId64 ", vgId:%d, type:%s, enable batch meta:%d", pHead->version, vgId, + TMSG_INFO(pHead->msgType), pRequest->enableBatchMeta); + if (!pRequest->enableBatchMeta && !pRequest->useSnapshot) { + SMqMetaRsp metaRsp = {0}; + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer + 1); + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); + goto end; + } + + if (!btMetaRsp.batchMetaReq) { + btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES); + btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t)); + } + fetchVer++; + + SMqMetaRsp tmpMetaRsp = {0}; + tmpMetaRsp.resMsgType = pHead->msgType; + tmpMetaRsp.metaRspLen = pHead->bodyLen; + tmpMetaRsp.metaRsp = pHead->body; + uint32_t len = 0; + tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code); + if (TSDB_CODE_SUCCESS != code) { + tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); + continue; + } + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); + SEncoder encoder = {0}; + tEncoderInit(&encoder, metaBuff, len); + code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); + if (code < 0) { + tEncoderClear(&encoder); + tqError("tmq extract meta from log, tEncodeMqMetaRsp error"); + continue; + } + taosArrayPush(btMetaRsp.batchMetaReq, &tBuf); + taosArrayPush(btMetaRsp.batchMetaLen, &tLen); + totalMetaRows++; + if ((taosArrayGetSize(btMetaRsp.batchMetaReq) >= tmqRowSize) || (taosGetTimestampMs() - st > 1000)) { + tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); + tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); + goto end; + } + continue; + } + + if (totalMetaRows > 0) { + tqOffsetResetToLog(&btMetaRsp.rspOffset, fetchVer); + tqSendBatchMetaPollRsp(pHandle, pMsg, pRequest, &btMetaRsp, vgId); goto end; } @@ -304,7 +364,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } end: - + tDeleteMqBatchMetaRsp(&btMetaRsp); tDeleteSTaosxRsp(&taosxRsp); return code; } @@ -352,6 +412,40 @@ static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int pMsgHead->walever = ever; } +int32_t tqSendBatchMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqBatchMetaRsp* pRsp, + int32_t vgId) { + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeMqBatchMetaRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + initMqRspHead(buf, TMQ_MSG_TYPE__POLL_BATCH_META_RSP, pReq->epoch, pReq->consumerId, sver, ever); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + tEncodeMqBatchMetaRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg resp = {.info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0}; + + tmsgSendRsp(&resp); + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type: batch meta, size:%ld offset type:%d", vgId, + pReq->consumerId, pReq->epoch, taosArrayGetSize(pRsp->batchMetaReq), pRsp->rspOffset.type); + + return 0; +} + int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId) { int32_t len = 0; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 62c3b06b65..c55745e5c5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -19,6 +19,13 @@ typedef struct SMStreamCheckpointReadyRspMsg { SMsgHead head; + int64_t streamId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t downstreamTaskId; + int32_t downstreamNodeId; + int64_t checkpointId; + int32_t transId; } SMStreamCheckpointReadyRspMsg; static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); @@ -39,20 +46,27 @@ static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) { pTask->id.streamId = pId->streamId; } -int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) { - int32_t vgId = pMeta->vgId; - STaskId taskId = {0}; +int32_t tqExpandStreamTask(SStreamTask* pTask) { + SStreamMeta* pMeta = pTask->pMeta; + int32_t vgId = pMeta->vgId; + STaskId taskId = {0}; + int64_t st = taosGetTimestampMs(); + + tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId); if (pTask->info.fillHistory) { taskId = replaceStreamTaskId(pTask); } - pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1); - if (pTask->pState == NULL) { - tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); - return -1; - } else { - tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); + // sink task does not need the pState + if (pTask->info.taskLevel != TASK_LEVEL__SINK) { + pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1); + if (pTask->pState == NULL) { + tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); + return -1; + } else { + tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); + } } if (pTask->info.fillHistory) { @@ -65,8 +79,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) .fillHistory = pTask->info.fillHistory, .winRange = pTask->dataRange.window, }; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - handle.vnode = pVnode; + handle.vnode = ((STQ*)pMeta->ahandle)->pVnode; handle.initTqReader = 1; } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) { handle.numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); @@ -83,6 +98,9 @@ int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode) qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); } + double el = (taosGetTimestampMs() - st) / 1000.0; + tqDebug("s-task:%s vgId:%d expand stream task completed, elapsed time:%.2fsec", pTask->id.idStr, vgId, el); + return TSDB_CODE_SUCCESS; } @@ -157,10 +175,9 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || *ppTask == NULL) { - tqError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped", vgId, req.taskId); + tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, req.taskId); rsp.code = TSDB_CODE_SUCCESS; streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); return rsp.code; } @@ -476,21 +493,27 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); if (pTask == NULL) { tqError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", vgId, req.downstreamTaskId); - return code; + return TSDB_CODE_STREAM_TASK_NOT_EXIST; } - tqDebug("vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", vgId, + tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); - streamProcessCheckpointReadyMsg(pTask); + streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); streamMetaReleaseTask(pMeta, pTask); { // send checkpoint ready rsp - SRpcMsg rsp = {.code = 0, .info = pMsg->info, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)}; - rsp.pCont = rpcMallocCont(rsp.contLen); - SMsgHead* pHead = rsp.pCont; - pHead->vgId = htonl(req.downstreamNodeId); + SMStreamCheckpointReadyRspMsg* pReadyRsp = rpcMallocCont(sizeof(SMStreamCheckpointReadyRspMsg)); + pReadyRsp->upstreamTaskId = req.upstreamTaskId; + pReadyRsp->upstreamNodeId = req.upstreamNodeId; + pReadyRsp->downstreamTaskId = req.downstreamTaskId; + pReadyRsp->downstreamNodeId = req.downstreamNodeId; + pReadyRsp->checkpointId = req.checkpointId; + pReadyRsp->streamId = req.streamId; + pReadyRsp->head.vgId = htonl(req.downstreamNodeId); + + SRpcMsg rsp = {.code = 0, .info = pMsg->info, .pCont = pReadyRsp, .contLen = sizeof(SMStreamCheckpointReadyRspMsg)}; tmsgSendRsp(&rsp); pMsg->info.handle = NULL; // disable auto rsp @@ -577,22 +600,11 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve return code; } -void tqStreamRmTaskBackend(SStreamMeta* pMeta, STaskId* id) { - char taskKey[128] = {0}; - sprintf(taskKey, "0x%" PRIx64 "-0x%x", id->streamId, (int32_t)id->taskId); - - char* path = taosMemoryCalloc(1, strlen(pMeta->path) + 128); - sprintf(path, "%s%s%s", pMeta->path, TD_DIRSEP, taskKey); - taosRemoveDir(path); - taosMemoryFree(path); - // do nothing -} - int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; - int32_t vgId = pMeta->vgId; - STaskId hTaskId = {0}; + int32_t vgId = pMeta->vgId; + STaskId hTaskId = {0}; tqDebug("vgId:%d receive msg to drop s-task:0x%x", vgId, pReq->taskId); streamMetaWLock(pMeta); @@ -608,6 +620,7 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen hTaskId.taskId = pTask->hTaskInfo.id.taskId; } + streamTaskSetRemoveBackendFiles(pTask); streamTaskClearHTaskAttr(pTask, pReq->resetRelHalt); streamMetaReleaseTask(pMeta, pTask); } @@ -634,10 +647,32 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen streamMetaWUnLock(pMeta); - tqStreamRmTaskBackend(pMeta, &id); +// tqStreamRemoveTaskBackend(pMeta, &id); return 0; } +int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen) { + SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; + + int32_t vgId = pMeta->vgId; + tqDebug("vgId:%d receive msg to update-checkpoint-info for s-task:0x%x", vgId, pReq->taskId); + + streamMetaWLock(pMeta); + + STaskId id = {.streamId = pReq->streamId, .taskId = pReq->taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + + if (ppTask != NULL && (*ppTask) != NULL) { + streamTaskUpdateTaskCheckpointInfo(*ppTask, pReq); + } else { // failed to get the task. + tqError("vgId:%d failed to locate the s-task:0x%x to update the checkpoint info, it may have been dropped already", + vgId, pReq->taskId); + } + + streamMetaWUnLock(pMeta); + return TSDB_CODE_SUCCESS; +} + static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { int32_t vgId = pMeta->vgId; int32_t code = 0; @@ -680,10 +715,8 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } if (isLeader && !tsDisableStream) { - streamMetaResetTaskStatus(pMeta); streamMetaWUnLock(pMeta); - - streamMetaStartAllTasks(pMeta); + streamMetaStartAllTasks(pMeta, tqExpandStreamTask); } else { streamMetaResetStartInfo(&pMeta->startInfo); streamMetaWUnLock(pMeta); @@ -701,10 +734,10 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t vgId = pMeta->vgId; if (type == STREAM_EXEC_T_START_ONE_TASK) { - streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId); + streamMetaStartOneTask(pMeta, pReq->streamId, pReq->taskId, tqExpandStreamTask); return 0; } else if (type == STREAM_EXEC_T_START_ALL_TASKS) { - streamMetaStartAllTasks(pMeta); + streamMetaStartAllTasks(pMeta, tqExpandStreamTask); return 0; } else if (type == STREAM_EXEC_T_RESTART_ALL_TASKS) { restartStreamTasks(pMeta, isLeader); @@ -820,13 +853,19 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { // clear flag set during do checkpoint, and open inputQ for all upstream tasks SStreamTaskState *pState = streamTaskGetStatus(pTask); if (pState->state == TASK_STATUS__CK) { + int32_t tranId = 0; + int64_t activeChkId = 0; + streamTaskGetActiveCheckpointInfo(pTask, &tranId, &activeChkId); + tqDebug("s-task:%s reset task status from checkpoint, current checkpointingId:%" PRId64 ", transId:%d", - pTask->id.idStr, pTask->chkInfo.checkpointingId, pTask->chkInfo.transId); + pTask->id.idStr, activeChkId, tranId); + streamTaskSetStatusReady(pTask); } else if (pState->state == TASK_STATUS__UNINIT) { tqDebug("s-task:%s start task by checking downstream tasks", pTask->id.idStr); ASSERT(pTask->status.downstreamReady == 0); - /*int32_t ret = */ streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); +// /*int32_t ret = */ streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); + tqStreamStartOneTaskAsync(pMeta, pTask->pMsgCb, pTask->id.streamId, pTask->id.taskId); } else { tqDebug("s-task:%s status:%s do nothing after receiving reset-task from mnode", pTask->id.idStr, pState->name); } @@ -837,6 +876,89 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { return TSDB_CODE_SUCCESS; } +int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d process retrieve checkpoint trigger, checkpointId:%" PRId64 + " from s-task:0x%x, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, pReq->checkpointId, (int32_t)pReq->downstreamTaskId, pReq->upstreamTaskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + tqDebug("s-task:0x%x recv retrieve checkpoint-trigger msg from downstream s-task:0x%x, checkpointId:%" PRId64, + pReq->upstreamTaskId, (int32_t)pReq->downstreamTaskId, pReq->checkpointId); + + if (pTask->status.downstreamReady != 1) { + tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", + pTask->id.idStr, (int32_t)pReq->downstreamTaskId); + + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS); + streamMetaReleaseTask(pMeta, pTask); + + return TSDB_CODE_SUCCESS; + } + + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state == TASK_STATUS__CK) { // recv the checkpoint-source/trigger already + int32_t transId = 0; + int64_t checkpointId = 0; + + streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId); + ASSERT (checkpointId == pReq->checkpointId); + + if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) { + // re-send the lost checkpoint-trigger msg to downstream task + tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, + (int32_t)pReq->downstreamTaskId, checkpointId, transId); + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_SUCCESS); + } else { // not send checkpoint-trigger yet, wait + int32_t recv = 0, total = 0; + streamTaskGetTriggerRecvStatus(pTask, &recv, &total); + + if (recv == total) { // add the ts info + tqWarn("s-task:%s all upstream send checkpoint-source/trigger, but not processed yet, wait", pTask->id.idStr); + } else { + tqWarn( + "s-task:%s not all upstream send checkpoint-source/trigger, total recv:%d/%d, wait for all upstream " + "sending checkpoint-source/trigger", + pTask->id.idStr, recv, total); + } + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); + } + } else { // upstream not recv the checkpoint-source/trigger till now + ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT); + tqWarn( + "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " + "upstream sending checkpoint-source/trigger", + pTask->id.idStr); + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); + } + + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SCheckpointTriggerRsp* pRsp = pMsg->pCont; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->taskId); + if (pTask == NULL) { + tqError( + "vgId:%d process retrieve checkpoint-trigger, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, pRsp->taskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%"PRId64", transId:%d", + pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); + + streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; +} + int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg) { SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg; @@ -927,9 +1049,10 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* msg, bool fromVnode) { SVResumeStreamTaskReq* pReq = (SVResumeStreamTaskReq*)msg; - SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); + + SStreamMeta* pMeta = fromVnode ? ((STQ*)handle)->pStreamMeta : handle; + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + int32_t code = tqProcessTaskResumeImpl(handle, pTask, sversion, pReq->igUntreated, fromVnode); if (code != 0) { return code; } @@ -956,5 +1079,16 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { - return doProcessDummyRspMsg(pMeta, pMsg); + SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->downstreamTaskId); + if (pTask == NULL) { + tqError("vgId:%d failed to acquire task:0x%x when handling checkpoint-ready msg, it may have been dropped", + pRsp->downstreamNodeId, pRsp->downstreamTaskId); + return TSDB_CODE_STREAM_TASK_NOT_EXIST; + } + + streamTaskProcessCheckpointReadyRsp(pTask, pRsp->upstreamTaskId, pRsp->checkpointId); + streamMetaReleaseTask(pMeta, pTask); + return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index f602ee5fe3..42df7f111e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -588,6 +588,13 @@ static void tsdbCacheDeleter(const void *key, size_t klen, void *value, void *ud tsdbCachePutBatch(pLastCol, key, klen, (SCacheFlushState *)ud); } + for (uint8_t i = 0; i < pLastCol->rowKey.numOfPKs; ++i) { + SValue *pValue = &pLastCol->rowKey.pks[i]; + if (IS_VAR_DATA_TYPE(pValue->type)) { + taosMemoryFree(pValue->pData); + } + } + if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type) /* && pLastCol->colVal.value.nData > 0*/) { taosMemoryFree(pLastCol->colVal.value.pData); } @@ -1072,6 +1079,8 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SLastCol *PToFree = pLastCol; if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) { + taosMemoryFreeClear(PToFree); + rocksdb_free(values_list[i]); continue; } @@ -1238,6 +1247,18 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx)); // 1. prepare last + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(&lRow, &tsdbRowKey); + + { + SLastUpdateCtx updateCtx = { + .lflag = LFLAG_LAST, + .tsdbRowKey = tsdbRowKey, + .colVal = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, + .val = lRow.pBlockData->aTSKEY[lRow.iRow]}))}; + taosArrayPush(ctxArray, &updateCtx); + } + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { @@ -1263,9 +1284,6 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo } // 2. prepare last row - STsdbRowKey tsdbRowKey = {0}; - tsdbRowGetKey(&lRow, &tsdbRowKey); - STSDBRowIter iter = {0}; tsdbRowIterOpen(&iter, &lRow, pTSchema); for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 96c0f2a4ba..4d62bc9632 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -17,13 +17,12 @@ // extern dependencies typedef struct { - STsdb *tsdb; - TFileSetArray *fsetArr; - TFileOpArray fopArray[1]; - - // SSkmInfo skmTb[1]; - // SSkmInfo skmRow[1]; + int32_t fid; + STFileSet *fset; +} SFileSetCommitInfo; +typedef struct { + STsdb *tsdb; int32_t minutes; int8_t precision; int32_t minRow; @@ -32,34 +31,34 @@ typedef struct { int32_t sttTrigger; int32_t szPage; int64_t compactVersion; + int64_t cid; + int64_t now; struct { - int64_t cid; - int64_t now; - TSKEY nextKey; - int32_t fid; - int32_t expLevel; - SDiskID did; - TSKEY minKey; - TSKEY maxKey; - STFileSet *fset; - TABLEID tbid[1]; - bool hasTSData; - bool skipTsRow; - SHashObj *pColCmprObj; + SFileSetCommitInfo *info; + + int32_t expLevel; + SDiskID did; + TSKEY minKey; + TSKEY maxKey; + TABLEID tbid[1]; + bool hasTSData; + + bool skipTsRow; + SHashObj *pColCmprObj; } ctx[1]; // reader TSttFileReaderArray sttReaderArray[1]; - // iter TTsdbIterArray dataIterArray[1]; SIterMerger *dataIterMerger; TTsdbIterArray tombIterArray[1]; SIterMerger *tombIterMerger; - // writer SFSetWriter *writer; + + TFileOpArray fopArray[1]; } SCommitter2; static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { @@ -74,8 +73,8 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { .maxRow = committer->maxRow, .szPage = committer->szPage, .cmprAlg = committer->cmprAlg, - .fid = committer->ctx->fid, - .cid = committer->ctx->cid, + .fid = committer->ctx->info->fid, + .cid = committer->cid, .did = committer->ctx->did, .level = 0, }; @@ -83,11 +82,11 @@ static int32_t tsdbCommitOpenWriter(SCommitter2 *committer) { if (committer->sttTrigger == 1) { config.toSttOnly = false; - if (committer->ctx->fset) { + if (committer->ctx->info->fset) { for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ftype++) { - if (committer->ctx->fset->farr[ftype] != NULL) { + if (committer->ctx->info->fset->farr[ftype] != NULL) { config.files[ftype].exist = true; - config.files[ftype].file = committer->ctx->fset->farr[ftype]->f[0]; + config.files[ftype].file = committer->ctx->info->fset->farr[ftype]->f[0]; } } } @@ -117,7 +116,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; - for (SRowInfo *row; (row = tsdbIterMergerGetData(committer->dataIterMerger)) != NULL;) { if (row->uid != committer->ctx->tbid->uid) { committer->ctx->tbid->suid = row->suid; @@ -132,7 +130,6 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { int64_t ts = TSDBROW_TS(&row->row); if (ts > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, ts); code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); continue; @@ -152,7 +149,8 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, numOfRow); + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " rows", TD_VID(committer->tsdb->pVnode), committer->ctx->info->fid, + numOfRow); } return code; } @@ -168,7 +166,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { } // do not need to write tomb data if there is no ts data - bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData); + bool skip = (committer->ctx->info->fset == NULL && !committer->ctx->hasTSData); committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -187,12 +185,8 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { if (record->ekey < committer->ctx->minKey) { // do nothing } else if (record->skey > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey); + // committer->ctx->nextKey = TMIN(record->skey, committer->ctx->nextKey); } else { - if (record->ekey > committer->ctx->maxKey) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, committer->ctx->maxKey + 1); - } - record->skey = TMAX(record->skey, committer->ctx->minKey); record->ekey = TMIN(record->ekey, committer->ctx->maxKey); @@ -211,27 +205,32 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), committer->ctx->fid, - numRecord); + tsdbDebug("vgId:%d fid:%d commit %" PRId64 " tomb records", TD_VID(committer->tsdb->pVnode), + committer->ctx->info->fid, numRecord); } return code; } +static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { + TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose); + return 0; +} + static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; ASSERT(TARRAY2_SIZE(committer->sttReaderArray) == 0); - if (committer->ctx->fset == NULL // - || committer->sttTrigger > 1 // - || TARRAY2_SIZE(committer->ctx->fset->lvlArr) == 0 // + if (committer->ctx->info->fset == NULL // + || committer->sttTrigger > 1 // + || TARRAY2_SIZE(committer->ctx->info->fset->lvlArr) == 0 // ) { return 0; } SSttLvl *lvl; - TARRAY2_FOREACH(committer->ctx->fset->lvlArr, lvl) { + TARRAY2_FOREACH(committer->ctx->info->fset->lvlArr, lvl) { STFileObj *fobj = NULL; TARRAY2_FOREACH(lvl->fobjArr, fobj) { SSttFileReader *sttReader; @@ -261,13 +260,17 @@ static int32_t tsdbCommitOpenReader(SCommitter2 *committer) { _exit: if (code) { + tsdbCommitCloseReader(committer); TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbCommitCloseReader(SCommitter2 *committer) { - TARRAY2_CLEAR(committer->sttReaderArray, tsdbSttFileReaderClose); +static int32_t tsdbCommitCloseIter(SCommitter2 *committer) { + tsdbIterMergerClose(&committer->tombIterMerger); + tsdbIterMergerClose(&committer->dataIterMerger); + TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose); + TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose); return 0; } @@ -289,7 +292,7 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { config.from->version = VERSION_MIN; config.from->key = (SRowKey){ .ts = committer->ctx->minKey, - .numOfPKs = 0, // TODO: support multiple primary keys + .numOfPKs = 0, }; code = tsdbIterOpen(&config, &iter); @@ -341,40 +344,26 @@ static int32_t tsdbCommitOpenIter(SCommitter2 *committer) { _exit: if (code) { + tsdbCommitCloseIter(committer); TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } return code; } -static int32_t tsdbCommitCloseIter(SCommitter2 *committer) { - tsdbIterMergerClose(&committer->tombIterMerger); - tsdbIterMergerClose(&committer->dataIterMerger); - TARRAY2_CLEAR(committer->tombIterArray, tsdbIterClose); - TARRAY2_CLEAR(committer->dataIterArray, tsdbIterClose); - return 0; -} - static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { int32_t code = 0; int32_t lino = 0; STsdb *tsdb = committer->tsdb; - int32_t fid = tsdbKeyFid(committer->ctx->nextKey, committer->minutes, committer->precision); - // check if can commit - tsdbFSCheckCommit(tsdb, fid); + tsdbFSCheckCommit(tsdb, committer->ctx->info->fid); - committer->ctx->fid = fid; - committer->ctx->expLevel = tsdbFidLevel(committer->ctx->fid, &tsdb->keepCfg, committer->ctx->now); - tsdbFidKeyRange(committer->ctx->fid, committer->minutes, committer->precision, &committer->ctx->minKey, + committer->ctx->expLevel = tsdbFidLevel(committer->ctx->info->fid, &tsdb->keepCfg, committer->now); + tsdbFidKeyRange(committer->ctx->info->fid, committer->minutes, committer->precision, &committer->ctx->minKey, &committer->ctx->maxKey); code = tfsAllocDisk(committer->tsdb->pVnode->pTfs, committer->ctx->expLevel, &committer->ctx->did); TSDB_CHECK_CODE(code, lino, _exit); tfsMkdirRecurAt(committer->tsdb->pVnode->pTfs, committer->tsdb->path, committer->ctx->did); - STFileSet fset = {.fid = committer->ctx->fid}; - committer->ctx->fset = &fset; - STFileSet **fsetPtr = TARRAY2_SEARCH(committer->fsetArr, &committer->ctx->fset, tsdbTFileSetCmprFn, TD_EQ); - committer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr; committer->ctx->tbid->suid = 0; committer->ctx->tbid->uid = 0; @@ -391,15 +380,13 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { code = tsdbCommitOpenWriter(committer); TSDB_CHECK_CODE(code, lino, _exit); - // reset nextKey - committer->ctx->nextKey = TSKEY_MAX; - _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); } else { tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode), - __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel); + __func__, committer->ctx->info->fid, committer->ctx->minKey, committer->ctx->maxKey, + committer->ctx->expLevel); } return code; } @@ -421,7 +408,7 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); } return code; } @@ -449,7 +436,220 @@ _exit: if (code) { TSDB_ERROR_LOG(TD_VID(committer->tsdb->pVnode), lino, code); } else { - tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->fid); + tsdbDebug("vgId:%d %s done, fid:%d", TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->info->fid); + } + return code; +} + +static int32_t tFileSetCommitInfoCompare(const void *arg1, const void *arg2) { + SFileSetCommitInfo *info1 = (SFileSetCommitInfo *)arg1; + SFileSetCommitInfo *info2 = (SFileSetCommitInfo *)arg2; + + if (info1->fid < info2->fid) { + return -1; + } else if (info1->fid > info2->fid) { + return 1; + } else { + return 0; + } +} + +static int32_t tFileSetCommitInfoPCompare(const void *arg1, const void *arg2) { + return tFileSetCommitInfoCompare(*(SFileSetCommitInfo **)arg1, *(SFileSetCommitInfo **)arg2); +} + +static uint32_t tFileSetCommitInfoHash(const void *arg) { + SFileSetCommitInfo *info = (SFileSetCommitInfo *)arg; + return MurmurHash3_32((const char *)&info->fid, sizeof(info->fid)); +} + +static int32_t tsdbCommitInfoDestroy(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + if (pTsdb->commitInfo) { + for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); + vHashDrop(pTsdb->commitInfo->ht, info); + tsdbTFileSetClear(&info->fset); + taosMemoryFree(info); + } + + vHashDestroy(&pTsdb->commitInfo->ht); + taosArrayDestroy(pTsdb->commitInfo->arr); + pTsdb->commitInfo->arr = NULL; + taosMemoryFreeClear(pTsdb->commitInfo); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoInit(STsdb *pTsdb) { + int32_t code = 0; + int32_t lino = 0; + + pTsdb->commitInfo = taosMemoryCalloc(1, sizeof(*pTsdb->commitInfo)); + if (pTsdb->commitInfo == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + + code = vHashInit(&pTsdb->commitInfo->ht, tFileSetCommitInfoHash, tFileSetCommitInfoCompare); + TSDB_CHECK_CODE(code, lino, _exit); + + pTsdb->commitInfo->arr = taosArrayInit(0, sizeof(SFileSetCommitInfo *)); + if (pTsdb->commitInfo->arr == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + +_exit: + if (code) { + tsdbCommitInfoDestroy(pTsdb); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoAdd(STsdb *tsdb, int32_t fid) { + int32_t code = 0; + int32_t lino = 0; + + SFileSetCommitInfo *tinfo; + + if ((tinfo = taosMemoryMalloc(sizeof(*tinfo))) == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + tinfo->fid = fid; + tinfo->fset = NULL; + + code = vHashPut(tsdb->commitInfo->ht, tinfo); + TSDB_CHECK_CODE(code, lino, _exit); + + if ((taosArrayPush(tsdb->commitInfo->arr, &tinfo)) == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } + taosArraySort(tsdb->commitInfo->arr, tFileSetCommitInfoPCompare); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbCommitInfoBuild(STsdb *tsdb) { + int32_t code = 0; + int32_t lino = 0; + + STFileSet *fset = NULL; + SRBTreeIter iter; + + code = tsdbCommitInfoInit(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + // scan time-series data + iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); + for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) { + STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn); + + // scan time-series data + STsdbRowKey from = { + .key.ts = INT64_MIN, + .key.numOfPKs = 0, + .version = INT64_MIN, + }; + for (;;) { + int64_t minKey, maxKey; + STbDataIter tbDataIter = {0}; + TSDBROW *row; + int32_t fid; + + tsdbTbDataIterOpen(pTbData, &from, 0, &tbDataIter); + if ((row = tsdbTbDataIterGet(&tbDataIter)) == NULL) { + break; + } + + fid = tsdbKeyFid(TSDBROW_TS(row), tsdb->keepCfg.days, tsdb->keepCfg.precision); + tsdbFidKeyRange(fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey); + + SFileSetCommitInfo *info; + SFileSetCommitInfo tinfo = { + .fid = fid, + }; + vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); + if (info == NULL) { + code = tsdbCommitInfoAdd(tsdb, fid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + from.key.ts = maxKey + 1; + } + } + + taosThreadMutexLock(&tsdb->mutex); + + // scan tomb data + if (tsdb->imem->nDel > 0) { + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + if (tsdbTFileSetIsEmpty(fset)) { + continue; + } + + SFileSetCommitInfo *info; + SFileSetCommitInfo tinfo = { + .fid = fset->fid, + }; + + // check if the file set already on the commit list + vHashGet(tsdb->commitInfo->ht, &tinfo, (void **)&info); + if (info != NULL) { + continue; + } + + int64_t minKey, maxKey; + bool hasDataToCommit = false; + tsdbFidKeyRange(fset->fid, tsdb->keepCfg.days, tsdb->keepCfg.precision, &minKey, &maxKey); + iter = tRBTreeIterCreate(tsdb->imem->tbDataTree, 1); + for (SRBTreeNode *node = tRBTreeIterNext(&iter); node; node = tRBTreeIterNext(&iter)) { + STbData *pTbData = TCONTAINER_OF(node, STbData, rbtn); + for (SDelData *pDelData = pTbData->pHead; pDelData; pDelData = pDelData->pNext) { + if (pDelData->sKey > maxKey || pDelData->eKey < minKey) { + continue; + } else { + hasDataToCommit = true; + if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) { + taosThreadMutexUnlock(&tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + break; + } + } + + if (hasDataToCommit) { + break; + } + } + } + } + + // begin tasks on file set + for (int i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + tsdbBeginTaskOnFileSet(tsdb, info->fid, &fset); + if (fset) { + tsdbTFileSetInitCopy(tsdb, fset, &info->fset); + } + } + + taosThreadMutexUnlock(&tsdb->mutex); + +_exit: + if (code) { + tsdbCommitInfoDestroy(tsdb); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -458,11 +658,7 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co int32_t code = 0; int32_t lino = 0; - memset(committer, 0, sizeof(committer[0])); - committer->tsdb = tsdb; - code = tsdbFSCreateCopySnapshot(tsdb->pFS, &committer->fsetArr); - TSDB_CHECK_CODE(code, lino, _exit); committer->minutes = tsdb->keepCfg.days; committer->precision = tsdb->keepCfg.precision; committer->minRow = info->info.config.tsdbCfg.minRows; @@ -471,21 +667,11 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co committer->sttTrigger = info->info.config.sttTrigger; committer->szPage = info->info.config.tsdbPageSize; committer->compactVersion = INT64_MAX; - committer->ctx->cid = tsdbFSAllocEid(tsdb->pFS); - committer->ctx->now = taosGetTimestampSec(); + committer->cid = tsdbFSAllocEid(tsdb->pFS); + committer->now = taosGetTimestampSec(); - committer->ctx->nextKey = tsdb->imem->minKey; - if (tsdb->imem->nDel > 0) { - SRBTreeIter iter[1] = {tRBTreeIterCreate(tsdb->imem->tbDataTree, 1)}; - - for (SRBTreeNode *node = tRBTreeIterNext(iter); node; node = tRBTreeIterNext(iter)) { - STbData *tbData = TCONTAINER_OF(node, STbData, rbtn); - - for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) { - committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey); - } - } - } + code = tsdbCommitInfoBuild(tsdb); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { @@ -516,14 +702,13 @@ static int32_t tsdbCloseCommitter(SCommitter2 *committer, int32_t eno) { TARRAY2_DESTROY(committer->sttReaderArray, NULL); TARRAY2_DESTROY(committer->fopArray, NULL); TARRAY2_DESTROY(committer->sttReaderArray, NULL); - tsdbFSDestroyCopySnapshot(&committer->fsetArr); _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, lino, - tstrerror(code), committer->ctx->cid); + tstrerror(code), committer->cid); } else { - tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->ctx->cid); + tsdbDebug("vgId:%d %s done, eid:%" PRId64, TD_VID(committer->tsdb->pVnode), __func__, committer->cid); } return code; } @@ -553,17 +738,18 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) { taosThreadMutexUnlock(&tsdb->mutex); tsdbUnrefMemTable(imem, NULL, true); } else { - SCommitter2 committer[1]; + SCommitter2 committer = {0}; - code = tsdbOpenCommitter(tsdb, info, committer); + code = tsdbOpenCommitter(tsdb, info, &committer); TSDB_CHECK_CODE(code, lino, _exit); - while (committer->ctx->nextKey != TSKEY_MAX) { - code = tsdbCommitFileSet(committer); + for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + committer.ctx->info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + code = tsdbCommitFileSet(&committer); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbCloseCommitter(committer, code); + code = tsdbCloseCommitter(&committer, code); TSDB_CHECK_CODE(code, lino, _exit); } @@ -580,22 +766,33 @@ int32_t tsdbCommitCommit(STsdb *tsdb) { int32_t code = 0; int32_t lino = 0; - if (tsdb->imem == NULL) goto _exit; + if (tsdb->imem) { + SMemTable *pMemTable = tsdb->imem; + + taosThreadMutexLock(&tsdb->mutex); + + if ((code = tsdbFSEditCommit(tsdb->pFS))) { + taosThreadMutexUnlock(&tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + tsdb->imem = NULL; + + for (int32_t i = 0; i < taosArrayGetSize(tsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(tsdb->commitInfo->arr, i); + if (info->fset) { + tsdbFinishTaskOnFileSet(tsdb, info->fid); + } + } - SMemTable *pMemTable = tsdb->imem; - taosThreadMutexLock(&tsdb->mutex); - code = tsdbFSEditCommit(tsdb->pFS); - if (code) { taosThreadMutexUnlock(&tsdb->mutex); - TSDB_CHECK_CODE(code, lino, _exit); + + tsdbCommitInfoDestroy(tsdb); + tsdbUnrefMemTable(pMemTable, NULL, true); } - tsdb->imem = NULL; - taosThreadMutexUnlock(&tsdb->mutex); - tsdbUnrefMemTable(pMemTable, NULL, true); _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(tsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__); } @@ -611,6 +808,16 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) { code = tsdbFSEditAbort(pTsdb->pFS); TSDB_CHECK_CODE(code, lino, _exit); + taosThreadMutexLock(&pTsdb->mutex); + for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) { + SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i); + if (info->fset) { + tsdbFinishTaskOnFileSet(pTsdb, info->fid); + } + } + taosThreadMutexUnlock(&pTsdb->mutex); + tsdbCommitInfoDestroy(pTsdb); + _exit: if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); @@ -618,4 +825,4 @@ _exit: tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); } return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c index acacf98f84..107677ce1e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c @@ -74,7 +74,7 @@ int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlo pBlock->file.stt->level = reader->config->file.stt->level; int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); @@ -130,7 +130,7 @@ static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFil TSDB_CHECK_CODE(code, lino, _exit); int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; + char *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey; if (writer->fd) { code = tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey); @@ -211,13 +211,13 @@ _exit: return code; } -int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock, - int32_t encryptAlgorithm, char* encryptKey) { +int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock, + int32_t encryptAlgorithm, char *encryptKey) { int32_t code = 0; int32_t lino = 0; code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength, - encryptAlgorithm, encryptKey); + encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); writer->ctx->offset += pDataBlock->dataLength; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 3ca26621a1..2345cc599b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -22,9 +22,6 @@ extern void remove_file(const char *fname); -#define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT -#define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) - typedef struct STFileHashEntry { struct STFileHashEntry *next; char fname[TSDB_FILENAME_LEN]; @@ -290,10 +287,8 @@ static int32_t commit_edit(STFileSystem *fs) { current_fname(fs->tsdb, current, TSDB_FCURRENT); if (fs->etype == TSDB_FEDIT_COMMIT) { current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); - } else if (fs->etype == TSDB_FEDIT_MERGE) { - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } else { - ASSERT(0); + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } int32_t code; @@ -324,11 +319,8 @@ static int32_t abort_edit(STFileSystem *fs) { if (fs->etype == TSDB_FEDIT_COMMIT) { current_fname(fs->tsdb, fname, TSDB_FCURRENT_C); - } else if (fs->etype == TSDB_FEDIT_MERGE) { - current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); } else { - tsdbError("vgId:%d %s failed since invalid etype:%d", TD_VID(fs->tsdb->pVnode), __func__, fs->etype); - ASSERT(0); + current_fname(fs->tsdb, fname, TSDB_FCURRENT_M); } int32_t code; @@ -767,9 +759,12 @@ extern int32_t tsdbStopAllCompTask(STsdb *tsdb); int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { STFileSystem *fs = pTsdb->pFS; - TARRAY2(int64_t) channelArr = {0}; + SArray *channelArray = taosArrayInit(0, sizeof(SVAChannelID)); + if (channelArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - taosThreadMutexLock(&fs->tsdb->mutex); + taosThreadMutexLock(&pTsdb->mutex); // disable pTsdb->bgTaskDisabled = true; @@ -777,20 +772,23 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) { // collect channel STFileSet *fset; TARRAY2_FOREACH(fs->fSetArr, fset) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { - TARRAY2_APPEND(&channelArr, fset->bgTaskChannel); - fset->bgTaskChannel = 0; + if (fset->channelOpened) { + taosArrayPush(channelArray, &fset->channel); + fset->channel = (SVAChannelID){0}; + fset->mergeScheduled = false; + tsdbFSSetBlockCommit(fset, false); + fset->channelOpened = false; } - fset->mergeScheduled = false; - tsdbFSSetBlockCommit(fset, false); } - taosThreadMutexUnlock(&fs->tsdb->mutex); + taosThreadMutexUnlock(&pTsdb->mutex); // destroy all channels - int64_t channel; - TARRAY2_FOREACH(&channelArr, channel) { vnodeAChannelDestroy(vnodeAsyncHandle[1], channel, true); } - TARRAY2_DESTROY(&channelArr, NULL); + for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) { + SVAChannelID *channel = taosArrayGet(channelArray, i); + vnodeAChannelDestroy(channel, true); + } + taosArrayDestroy(channelArray); #ifdef TD_ENTERPRISE tsdbStopAllCompTask(pTsdb); @@ -832,15 +830,10 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e int32_t lino; char current_t[TSDB_FILENAME_LEN]; - switch (etype) { - case TSDB_FEDIT_COMMIT: - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); - break; - case TSDB_FEDIT_MERGE: - current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); - break; - default: - ASSERT(0); + if (etype == TSDB_FEDIT_COMMIT) { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_C); + } else { + current_fname(fs->tsdb, current_t, TSDB_FCURRENT_M); } tsem_wait(&fs->canEdit); @@ -932,8 +925,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { arg->tsdb = fs->tsdb; arg->fid = fset->fid; - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, - NULL); + code = vnodeAsync(&fset->channel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree, arg, NULL); TSDB_CHECK_CODE(code, lino, _exit); fset->mergeScheduled = true; } @@ -946,33 +938,6 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { } } - // clear empty level and fset - int32_t i = 0; - while (i < TARRAY2_SIZE(fs->fSetArr)) { - STFileSet *fset = TARRAY2_GET(fs->fSetArr, i); - - int32_t j = 0; - while (j < TARRAY2_SIZE(fset->lvlArr)) { - SSttLvl *lvl = TARRAY2_GET(fset->lvlArr, j); - - if (TARRAY2_SIZE(lvl->fobjArr) == 0) { - TARRAY2_REMOVE(fset->lvlArr, j, tsdbSttLvlClear); - } else { - j++; - } - } - - if (tsdbTFileSetIsEmpty(fset)) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) { - vnodeAChannelDestroy(vnodeAsyncHandle[1], fset->bgTaskChannel, false); - fset->bgTaskChannel = 0; - } - TARRAY2_REMOVE(fs->fSetArr, i, tsdbTFileSetClear); - } else { - i++; - } - } - _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(fs->tsdb->pVnode), __func__, lino, tstrerror(code)); @@ -1211,3 +1176,47 @@ _out: } int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); } + +int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset) { + int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; + + tsdbFSGetFSet(tsdb->pFS, fid, fset); + if (sttTrigger == 1 && (*fset)) { + for (;;) { + if ((*fset)->taskRunning) { + (*fset)->numWaitTask++; + + taosThreadCondWait(&(*fset)->beginTask, &tsdb->mutex); + + tsdbFSGetFSet(tsdb->pFS, fid, fset); + ASSERT(fset != NULL); + + (*fset)->numWaitTask--; + ASSERT((*fset)->numWaitTask >= 0); + } else { + (*fset)->taskRunning = true; + break; + } + } + tsdbInfo("vgId:%d begin task on file set:%d", TD_VID(tsdb->pVnode), fid); + } + + return 0; +} + +int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid) { + int16_t sttTrigger = tsdb->pVnode->config.sttTrigger; + if (sttTrigger == 1) { + STFileSet *fset = NULL; + tsdbFSGetFSet(tsdb->pFS, fid, &fset); + if (fset != NULL && fset->taskRunning) { + fset->taskRunning = false; + if (fset->numWaitTask > 0) { + taosThreadCondSignal(&fset->beginTask); + } + tsdbInfo("vgId:%d finish task on file set:%d", TD_VID(tsdb->pVnode), fid); + } + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 7099ce8b26..6938efde3c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -24,7 +24,9 @@ extern "C" { typedef enum { TSDB_FEDIT_COMMIT = 1, // - TSDB_FEDIT_MERGE + TSDB_FEDIT_MERGE, + TSDB_FEDIT_COMPACT, + TSDB_FEDIT_RETENTION, } EFEditT; typedef enum { @@ -59,6 +61,8 @@ int32_t tsdbFSEditAbort(STFileSystem *fs); // other int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset); int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid); +int32_t tsdbBeginTaskOnFileSet(STsdb *tsdb, int32_t fid, STFileSet **fset); +int32_t tsdbFinishTaskOnFileSet(STsdb *tsdb, int32_t fid); // utils int32_t save_fs(const TFileSetArray *arr, const char *fname); int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype); @@ -72,10 +76,6 @@ struct STFileSystem { EFEditT etype; TFileSetArray fSetArr[1]; TFileSetArray fSetArrTmp[1]; - - // background task queue - bool stop; - int64_t taskid; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 0820f53117..598ca0aec3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -456,13 +456,14 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) { TARRAY2_INIT(fset[0]->lvlArr); // background task queue - fset[0]->bgTaskChannel = 0; - fset[0]->mergeScheduled = false; + taosThreadCondInit(&(*fset)->beginTask, NULL); + (*fset)->taskRunning = false; + (*fset)->numWaitTask = 0; // block commit variables taosThreadCondInit(&fset[0]->canCommit, NULL); - fset[0]->numWaitCommit = 0; - fset[0]->blockCommit = false; + (*fset)->numWaitCommit = 0; + (*fset)->blockCommit = false; return 0; } @@ -598,21 +599,19 @@ int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr) { return 0; } -int32_t tsdbTFileSetClear(STFileSet **fset) { - if (!fset[0]) return 0; +void tsdbTFileSetClear(STFileSet **fset) { + if (fset && *fset) { + for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { + if ((*fset)->farr[ftype] == NULL) continue; + tsdbTFileObjUnref((*fset)->farr[ftype]); + } - for (tsdb_ftype_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) { - if (fset[0]->farr[ftype] == NULL) continue; - tsdbTFileObjUnref(fset[0]->farr[ftype]); + TARRAY2_DESTROY((*fset)->lvlArr, tsdbSttLvlClear); + + taosThreadCondDestroy(&(*fset)->beginTask); + taosThreadCondDestroy(&(*fset)->canCommit); + taosMemoryFreeClear(*fset); } - - TARRAY2_DESTROY(fset[0]->lvlArr, tsdbSttLvlClear); - - taosThreadCondDestroy(&fset[0]->canCommit); - taosMemoryFree(fset[0]); - fset[0] = NULL; - - return 0; } int32_t tsdbTFileSetRemove(STFileSet *fset) { @@ -665,6 +664,12 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) { } int32_t tsdbTFileSetOpenChannel(STFileSet *fset) { - if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) return 0; - return vnodeAChannelInit(vnodeAsyncHandle[1], &fset->bgTaskChannel); + int32_t code; + if (!fset->channelOpened) { + if ((code = vnodeAChannelInit(2, &fset->channel))) { + return code; + } + fset->channelOpened = true; + } + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 86db3b01c7..4b99a46dc7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -42,7 +42,7 @@ typedef enum { int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset); int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset); -int32_t tsdbTFileSetClear(STFileSet **fset); +void tsdbTFileSetClear(STFileSet **fset); int32_t tsdbTFileSetRemove(STFileSet *fset); int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset, @@ -90,9 +90,15 @@ struct STFileSet { STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array - // background task channel - int64_t bgTaskChannel; - bool mergeScheduled; + // background task + bool channelOpened; + SVAChannelID channel; + bool mergeScheduled; + + // sttTrigger = 1 + TdThreadCond beginTask; + bool taskRunning; + int32_t numWaitTask; // block commit variables TdThreadCond canCommit; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index cde74f5e36..7d8b0a6c8b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -44,7 +44,7 @@ static const struct { }; void remove_file(const char *fname) { - taosRemoveFile(fname); + (void)taosRemoveFile(fname); tsdbInfo("file:%s is removed", fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index 971020e7d6..f2d4ab2cb2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -90,6 +90,11 @@ _exit: return code; } +static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { + TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose); + return 0; +} + static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { int32_t code = 0; int32_t lino = 0; @@ -220,8 +225,10 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { code = tsdbSttFileReaderOpen(fobj->fname, &config, &reader); TSDB_CHECK_CODE(code, lino, _exit); - code = TARRAY2_APPEND(merger->sttReaderArr, reader); - TSDB_CHECK_CODE(code, lino, _exit); + if ((code = TARRAY2_APPEND(merger->sttReaderArr, reader))) { + tsdbSttFileReaderClose(&reader); + TSDB_CHECK_CODE(code, lino, _exit); + } } } @@ -232,6 +239,7 @@ static int32_t tsdbMergeFileSetBeginOpenReader(SMerger *merger) { _exit: if (code) { + tsdbMergeFileSetEndCloseReader(merger); TSDB_ERROR_LOG(TD_VID(merger->tsdb->pVnode), lino, code); } return code; @@ -375,11 +383,6 @@ static int32_t tsdbMergeFileSetEndCloseIter(SMerger *merger) { return 0; } -static int32_t tsdbMergeFileSetEndCloseReader(SMerger *merger) { - TARRAY2_CLEAR(merger->sttReaderArr, tsdbSttFileReaderClose); - return 0; -} - static int32_t tsdbMergeFileSetEnd(SMerger *merger) { int32_t code = 0; int32_t lino = 0; @@ -593,5 +596,6 @@ _exit: exit(EXIT_FAILURE); } tsdbTFileSetClear(&merger->fset); + taosMemoryFree(arg); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index ea3d285880..f9fede1d9b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -30,6 +30,14 @@ int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) { return 0; } +int64_t tsdbGetEarliestTs(STsdb *pTsdb) { + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + + int64_t now = taosGetTimestamp(pCfg->precision); + int64_t ts = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick + return ts; +} + /** * @brief * diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index dd79726762..e1cfcba070 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -226,14 +226,10 @@ static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. static STimeWindow updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow) { - STsdbKeepCfg* pCfg = &pTsdb->keepCfg; - - int64_t now = taosGetTimestamp(pCfg->precision); - int64_t earilyTs = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick - + int64_t earlyTs = tsdbGetEarliestTs(pTsdb); STimeWindow win = *pWindow; - if (win.skey < earilyTs) { - win.skey = earilyTs; + if (win.skey < earlyTs) { + win.skey = earlyTs; } return win; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f4344296b4..3d53d1ada3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -24,8 +24,8 @@ typedef struct { int64_t now; int64_t cid; - TFileSetArray *fsetArr; - TFileOpArray fopArr[1]; + STFileSet *fset; + TFileOpArray fopArr; } SRTNer; static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { @@ -35,7 +35,7 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { .of = fobj->f[0], }; - return TARRAY2_APPEND(rtner->fopArr, op); + return TARRAY2_APPEND(&rtner->fopArr, op); } static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { @@ -71,7 +71,7 @@ static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFi _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); if (fdFrom) taosCloseFile(&fdFrom); if (fdTo) taosCloseFile(&fdTo); } @@ -108,7 +108,7 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); taosCloseFile(&fdFrom); taosCloseFile(&fdTo); } @@ -128,7 +128,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -152,7 +152,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // do copy the file @@ -167,7 +167,7 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } return code; } @@ -176,80 +176,51 @@ typedef struct { STsdb *tsdb; int64_t now; int32_t fid; + bool s3Migrate; } SRtnArg; -static int32_t tsdbDoRetentionBegin(SRtnArg *arg, SRTNer *rtner) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *tsdb = arg->tsdb; - - rtner->tsdb = tsdb; - rtner->szPage = tsdb->pVnode->config.tsdbPageSize; - rtner->now = arg->now; - rtner->cid = tsdbFSAllocEid(tsdb->pFS); - - code = tsdbFSCreateCopySnapshot(tsdb->pFS, &rtner->fsetArr); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - } else { - tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); - } - return code; -} - static int32_t tsdbDoRetentionEnd(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; - if (TARRAY2_SIZE(rtner->fopArr) == 0) goto _exit; - - code = tsdbFSEditBegin(rtner->tsdb->pFS, rtner->fopArr, TSDB_FEDIT_MERGE); - TSDB_CHECK_CODE(code, lino, _exit); - - taosThreadMutexLock(&rtner->tsdb->mutex); - - code = tsdbFSEditCommit(rtner->tsdb->pFS); - if (code) { - taosThreadMutexUnlock(&rtner->tsdb->mutex); + if (TARRAY2_SIZE(&rtner->fopArr) > 0) { + code = tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION); TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadMutexLock(&rtner->tsdb->mutex); + + code = tsdbFSEditCommit(rtner->tsdb->pFS); + if (code) { + taosThreadMutexUnlock(&rtner->tsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosThreadMutexUnlock(&rtner->tsdb->mutex); + + TARRAY2_DESTROY(&rtner->fopArr, NULL); } - taosThreadMutexUnlock(&rtner->tsdb->mutex); - - TARRAY2_DESTROY(rtner->fopArr, NULL); - _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } else { tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__); } - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); return code; } -static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { +static int32_t tsdbDoRetention(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; STFileObj *fobj = NULL; + STFileSet *fset = rtner->fset; int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->now); if (expLevel < 0) { // remove the fileset for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - /* - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { - code = tsdbRemoveFileObjectS3(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } else {*/ code = tsdbDoRemoveFileObject(rtner, fobj); TSDB_CHECK_CODE(code, lino, _exit); - //} } SSttLvl *lvl; @@ -275,10 +246,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { if (fobj == NULL) continue; if (fobj->f->did.level == did.level) { - /* - code = tsdbCheckMigrateS3(rtner, fobj, ftype, &did); - TSDB_CHECK_CODE(code, lino, _exit); - */ continue; } @@ -306,91 +273,110 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(rtner->tsdb->pVnode), __func__, code, lino); } return code; } -static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } +static void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); } -static int32_t tsdbDoRetentionAsync(void *arg) { +static int32_t tsdbDoS3Migrate(SRTNer *rtner); + +static int32_t tsdbRetention(void *arg) { int32_t code = 0; int32_t lino = 0; - SRTNer rtner[1] = {0}; - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); + SRtnArg *rtnArg = (SRtnArg *)arg; + STsdb *pTsdb = rtnArg->tsdb; + SVnode *pVnode = pTsdb->pVnode; + STFileSet *fset = NULL; + SRTNer rtner = { + .tsdb = pTsdb, + .szPage = pVnode->config.tsdbPageSize, + .now = rtnArg->now, + .cid = tsdbFSAllocEid(pTsdb->pFS), + }; - STFileSet *fset; - TARRAY2_FOREACH(rtner->fsetArr, fset) { - if (fset->fid != ((SRtnArg *)arg)->fid) continue; + // begin task + taosThreadMutexLock(&pTsdb->mutex); + tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset); + if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) { + taosThreadMutexUnlock(&pTsdb->mutex); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosThreadMutexUnlock(&pTsdb->mutex); - code = tsdbDoRetentionOnFileSet(rtner, fset); + // do retention + if (rtner.fset) { + if (rtnArg->s3Migrate) { + code = tsdbDoS3Migrate(&rtner); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbDoRetention(&rtner); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbDoRetentionEnd(&rtner); TSDB_CHECK_CODE(code, lino, _exit); } - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - _exit: - if (code) { - if (TARRAY2_DATA(rtner->fopArr)) { - TARRAY2_DESTROY(rtner->fopArr, NULL); - } - TFileSetArray **fsetArr = &rtner->fsetArr; - if (fsetArr[0]) { - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); - } + if (rtner.fset) { + taosThreadMutexLock(&pTsdb->mutex); + tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid); + taosThreadMutexUnlock(&pTsdb->mutex); + } - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); + // clear resources + tsdbTFileSetClear(&rtner.fset); + TARRAY2_DESTROY(&rtner.fopArr, NULL); + taosMemoryFree(arg); + if (code) { + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(((SRtnArg *)arg)->tsdb->pVnode), __func__, code, lino); } return code; } -int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { +static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, int64_t now, bool s3Migrate) { int32_t code = 0; - - taosThreadMutexLock(&tsdb->mutex); - - if (tsdb->bgTaskDisabled) { - taosThreadMutexUnlock(&tsdb->mutex); - return 0; - } + int32_t lino = 0; STFileSet *fset; - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - code = tsdbTFileSetOpenChannel(fset); - if (code) { - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - taosThreadMutexUnlock(&tsdb->mutex); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (!tsdb->bgTaskDisabled) { + TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { + code = tsdbTFileSetOpenChannel(fset); + TSDB_CHECK_CODE(code, lino, _exit); - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; + SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); + if (arg == NULL) { + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); + } - if (sync) { - code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, - tsdbFreeRtnArg, arg, NULL); - } else { - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoRetentionAsync, - tsdbFreeRtnArg, arg, NULL); - } - if (code) { - tsdbFreeRtnArg(arg); - taosThreadMutexUnlock(&tsdb->mutex); - return code; + arg->tsdb = tsdb; + arg->now = now; + arg->fid = fset->fid; + arg->s3Migrate = s3Migrate; + + if ((code = vnodeAsync(&fset->channel, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg, NULL))) { + taosMemoryFree(arg); + TSDB_CHECK_CODE(code, lino, _exit); + } } } - taosThreadMutexUnlock(&tsdb->mutex); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed, code:%d, line:%d", TD_VID(tsdb->pVnode), __func__, code, lino); + } + return code; +} +int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) { + int32_t code = 0; + taosThreadMutexLock(&tsdb->mutex); + code = tsdbAsyncRetentionImpl(tsdb, now, false); + taosThreadMutexUnlock(&tsdb->mutex); return code; } @@ -462,7 +448,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -486,7 +472,7 @@ static int32_t tsdbMigrateDataFileLCS3(SRTNer *rtner, const STFileObj *fobj, int }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); char fname[TSDB_FILENAME_LEN]; @@ -566,7 +552,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 .of = fobj->f[0], }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); // create new @@ -590,7 +576,7 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, int64 }, }; - code = TARRAY2_APPEND(rtner->fopArr, op); + code = TARRAY2_APPEND(&rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); char fname[TSDB_FILENAME_LEN]; @@ -653,10 +639,11 @@ _exit: return code; } -static int32_t tsdbDoS3MigrateOnFileSet(SRTNer *rtner, STFileSet *fset) { +static int32_t tsdbDoS3Migrate(SRTNer *rtner) { int32_t code = 0; int32_t lino = 0; + STFileSet *fset = rtner->fset; STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; if (!fobj) return code; @@ -720,41 +707,7 @@ _exit: return code; } -static int32_t tsdbDoS3MigrateAsync(void *arg) { - int32_t code = 0; - int32_t lino = 0; - SRTNer rtner[1] = {0}; - - code = tsdbDoRetentionBegin(arg, rtner); - TSDB_CHECK_CODE(code, lino, _exit); - - STFileSet *fset; - TARRAY2_FOREACH(rtner->fsetArr, fset) { - if (fset->fid != ((SRtnArg *)arg)->fid) continue; - - code = tsdbDoS3MigrateOnFileSet(rtner, fset); - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbDoRetentionEnd(rtner); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - if (TARRAY2_DATA(rtner->fopArr)) { - TARRAY2_DESTROY(rtner->fopArr, NULL); - } - TFileSetArray **fsetArr = &rtner->fsetArr; - if (fsetArr[0]) { - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); - } - - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); - } - return code; -} - -int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) { +int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) { int32_t code = 0; extern int8_t tsS3EnabledCfg; @@ -772,45 +725,7 @@ int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) { } taosThreadMutexLock(&tsdb->mutex); - - if (tsdb->bgTaskDisabled) { - taosThreadMutexUnlock(&tsdb->mutex); - return 0; - } - - STFileSet *fset; - TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) { - code = tsdbTFileSetOpenChannel(fset); - if (code) { - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - - SRtnArg *arg = taosMemoryMalloc(sizeof(*arg)); - if (arg == NULL) { - taosThreadMutexUnlock(&tsdb->mutex); - return TSDB_CODE_OUT_OF_MEMORY; - } - - arg->tsdb = tsdb; - arg->now = now; - arg->fid = fset->fid; - - if (sync) { - code = vnodeAsyncC(vnodeAsyncHandle[0], tsdb->pVnode->commitChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, - tsdbFreeRtnArg, arg, NULL); - } else { - code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_LOW, tsdbDoS3MigrateAsync, - tsdbFreeRtnArg, arg, NULL); - } - if (code) { - tsdbFreeRtnArg(arg); - taosThreadMutexUnlock(&tsdb->mutex); - return code; - } - } - + code = tsdbAsyncRetentionImpl(tsdb, now, true); taosThreadMutexUnlock(&tsdb->mutex); - return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c index 9dae9bdd36..bc7bf5449e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapInfo.c @@ -321,10 +321,10 @@ int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartLis if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err; if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err; if (tDecodeI64(&decoder, &reserved64) < 0) goto _err; - TARRAY2_APPEND(iList, r); + if (TARRAY2_APPEND(iList, r)) goto _err; } } - TARRAY2_APPEND(pList, p); + if (TARRAY2_APPEND(pList, p)) goto _err; p = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 0d36f9648b..df89746c31 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -57,6 +57,20 @@ struct STsdbSnapReader { STombBlock tombBlock[1]; }; +static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { + int32_t code = 0; + int32_t lino = 0; + + TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); + tsdbDataFileReaderClose(&reader->dataReader); + +_exit: + if (code) { + TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); + } + return code; +} + static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { int32_t code = 0; int32_t lino = 0; @@ -100,27 +114,16 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) { code = tsdbSttFileReaderOpen(fobj->fname, &config, &sttReader); TSDB_CHECK_CODE(code, lino, _exit); - code = TARRAY2_APPEND(reader->sttReaderArr, sttReader); - TSDB_CHECK_CODE(code, lino, _exit); + if ((code = TARRAY2_APPEND(reader->sttReaderArr, sttReader))) { + tsdbSttFileReaderClose(&sttReader); + TSDB_CHECK_CODE(code, lino, _exit); + } } } _exit: if (code) { - TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); - } - return code; -} - -static int32_t tsdbSnapReadFileSetCloseReader(STsdbSnapReader* reader) { - int32_t code = 0; - int32_t lino = 0; - - TARRAY2_CLEAR(reader->sttReaderArr, tsdbSttFileReaderClose); - tsdbDataFileReaderClose(&reader->dataReader); - -_exit: - if (code) { + tsdbSnapReadFileSetCloseReader(reader); TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c index ab0575cb53..e4f48745e7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c @@ -141,6 +141,7 @@ static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) { _exit: if (code) { + tsdbSnapRAWReadFileSetCloseReader(reader); TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino); } return code; @@ -546,7 +547,7 @@ static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbD int32_t lino = 0; int32_t encryptAlgorithm = writer->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm; - char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey; + char* encryptKey = writer->tsdb->pVnode->config.tsdbCfg.encryptKey; code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr, encryptAlgorithm, encryptKey); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index 478778319d..f1a087e3c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -896,7 +896,7 @@ static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) { char fname[TSDB_FILENAME_LEN]; tsdbTFileName(writer->config->tsdb, writer->file, fname); tsdbCloseFile(&writer->fd); - taosRemoveFile(fname); + (void)taosRemoveFile(fname); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil2.c b/source/dnode/vnode/src/tsdb/tsdbUtil2.c index 6df0528f1c..9b88500ad1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil2.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil2.c @@ -56,7 +56,8 @@ int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record) { for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) { SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * idx, &tombBlock->buffers[i]); - tBufferGetI64(&br, &record->data[i]); + int32_t code = tBufferGetI64(&br, &record->data[i]); + if (code) return code; } return 0; } @@ -182,7 +183,8 @@ int32_t tStatisBlockPut(STbStatisBlock *block, SRowInfo *row, int32_t maxRecords if (block->numOfRecords > 0) { int64_t lastUid; SBufferReader br = BUFFER_READER_INITIALIZER(sizeof(int64_t) * (block->numOfRecords - 1), &block->uids); - tBufferGetI64(&br, &lastUid); + int32_t code = tBufferGetI64(&br, &lastUid); + if (code) return code; if (lastUid == row->uid) { return tStatisBlockUpdate(block, row); diff --git a/source/dnode/vnode/src/vnd/vnodeAsync.c b/source/dnode/vnode/src/vnd/vnodeAsync.c index 313603b19f..6d6533463b 100644 --- a/source/dnode/vnode/src/vnd/vnodeAsync.c +++ b/source/dnode/vnode/src/vnd/vnodeAsync.c @@ -16,6 +16,7 @@ #include "vnd.h" #include "vnodeHash.h" +typedef struct SVAsync SVAsync; typedef struct SVATask SVATask; typedef struct SVAChannel SVAChannel; @@ -54,7 +55,7 @@ struct SVATask { int32_t priorScore; SVAChannel *channel; int32_t (*execute)(void *); - void (*complete)(void *); + void (*cancel)(void *); void *arg; EVATaskState state; @@ -67,6 +68,11 @@ struct SVATask { struct SVATask *next; }; +typedef struct { + void (*cancel)(void *); + void *arg; +} SVATaskCancelInfo; + #define VATASK_PIORITY(task_) ((task_)->priority - ((task_)->priorScore / 4)) // async channel @@ -112,6 +118,10 @@ struct SVAsync { SVHashTable *taskTable; }; +SVAsync *vnodeAsyncs[3]; +#define MIN_ASYNC_ID 1 +#define MAX_ASYNC_ID (sizeof(vnodeAsyncs) / sizeof(vnodeAsyncs[0]) - 1) + static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { int32_t ret; @@ -160,11 +170,6 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { } async->numTasks--; - // call complete callback - if (task->complete) { - task->complete(task->arg); - } - if (task->numWait == 0) { taosThreadCondDestroy(&task->waitCond); taosMemoryFree(task); @@ -176,7 +181,7 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) { return 0; } -static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { +static int32_t vnodeAsyncCancelAllTasks(SVAsync *async, SArray *cancelArray) { while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] || async->queue[2].next != &async->queue[2]) { for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { @@ -184,6 +189,12 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { SVATask *task = async->queue[i].next; task->prev->next = task->next; task->next->prev = task->prev; + if (task->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }); + } vnodeAsyncTaskDone(async, task); } } @@ -194,6 +205,7 @@ static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) { static void *vnodeAsyncLoop(void *arg) { SVWorker *worker = (SVWorker *)arg; SVAsync *async = worker->async; + SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); setThreadName(async->label); @@ -209,12 +221,12 @@ static void *vnodeAsyncLoop(void *arg) { for (;;) { if (async->stop || worker->workerId >= async->numWorkers) { if (async->stop) { // cancel all tasks - vnodeAsyncCancelAllTasks(async); + vnodeAsyncCancelAllTasks(async, cancelArray); } worker->state = EVA_WORKER_STATE_STOP; async->numLaunchWorkers--; taosThreadMutexUnlock(&async->mutex); - return NULL; + goto _exit; } for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) { @@ -259,6 +271,12 @@ static void *vnodeAsyncLoop(void *arg) { worker->runningTask->execute(worker->runningTask->arg); } +_exit: + for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) { + SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i); + cancel->cancel(cancel->arg); + } + taosArrayDestroy(cancelArray); return NULL; } @@ -294,7 +312,7 @@ static int32_t vnodeAsyncChannelCompare(const void *obj1, const void *obj2) { return 0; } -int32_t vnodeAsyncInit(SVAsync **async, char *label) { +static int32_t vnodeAsyncInit(SVAsync **async, const char *label) { int32_t ret; if (async == NULL) { @@ -360,7 +378,7 @@ int32_t vnodeAsyncInit(SVAsync **async, char *label) { return 0; } -int32_t vnodeAsyncDestroy(SVAsync **async) { +static int32_t vnodeAsyncDestroy(SVAsync **async) { if ((*async) == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -433,20 +451,39 @@ static int32_t vnodeAsyncLaunchWorker(SVAsync *async) { return 0; } -#ifdef BUILD_NO_CALL -int32_t vnodeAsync(SVAsync *async, EVAPriority priority, int32_t (*execute)(void *), void (*complete)(void *), - void *arg, int64_t *taskId) { - return vnodeAsyncC(async, 0, priority, execute, complete, arg, taskId); -} -#endif +int32_t vnodeAsyncOpen(int32_t numOfThreads) { + int32_t code = 0; + int32_t lino = 0; -int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int32_t (*execute)(void *), - void (*complete)(void *), void *arg, int64_t *taskId) { - if (async == NULL || execute == NULL || channelId < 0) { + // vnode-commit + code = vnodeAsyncInit(&vnodeAsyncs[1], "vnode-commit"); + TSDB_CHECK_CODE(code, lino, _exit); + vnodeAsyncSetWorkers(1, numOfThreads); + + // vnode-merge + code = vnodeAsyncInit(&vnodeAsyncs[2], "vnode-merge"); + TSDB_CHECK_CODE(code, lino, _exit); + vnodeAsyncSetWorkers(2, numOfThreads); + +_exit: + return 0; +} + +int32_t vnodeAsyncClose() { + vnodeAsyncDestroy(&vnodeAsyncs[1]); + vnodeAsyncDestroy(&vnodeAsyncs[2]); + return 0; +} + +int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*execute)(void *), void (*cancel)(void *), + void *arg, SVATaskID *taskID) { + if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || execute == NULL || + channelID->id < 0) { return TSDB_CODE_INVALID_PARA; } - int64_t id; + int64_t id; + SVAsync *async = vnodeAsyncs[channelID->async]; // create task object SVATask *task = (SVATask *)taosMemoryCalloc(1, sizeof(SVATask)); @@ -457,7 +494,7 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int task->priority = priority; task->priorScore = 0; task->execute = execute; - task->complete = complete; + task->cancel = cancel; task->arg = arg; task->state = EVA_TASK_STATE_WAITTING; task->numWait = 0; @@ -466,10 +503,12 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int // schedule task taosThreadMutexLock(&async->mutex); - if (channelId == 0) { + if (channelID->id == 0) { task->channel = NULL; } else { - SVAChannel channel = {.channelId = channelId}; + SVAChannel channel = { + .channelId = channelID->id, + }; vHashGet(async->channelTable, &channel, (void **)&task->channel); if (task->channel == NULL) { taosThreadMutexUnlock(&async->mutex); @@ -540,20 +579,24 @@ int32_t vnodeAsyncC(SVAsync *async, int64_t channelId, EVAPriority priority, int taosThreadMutexUnlock(&async->mutex); - if (taskId != NULL) { - *taskId = id; + if (taskID != NULL) { + taskID->async = channelID->async; + taskID->id = id; } return 0; } -int32_t vnodeAWait(SVAsync *async, int64_t taskId) { - if (async == NULL || taskId <= 0) { +int32_t vnodeAWait(SVATaskID *taskID) { + if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[taskID->async]; SVATask *task = NULL; - SVATask task2 = {.taskId = taskId}; + SVATask task2 = { + .taskId = taskID->id, + }; taosThreadMutexLock(&async->mutex); @@ -574,21 +617,27 @@ int32_t vnodeAWait(SVAsync *async, int64_t taskId) { return 0; } -int32_t vnodeACancel(SVAsync *async, int64_t taskId) { - if (async == NULL) { +int32_t vnodeACancel(SVATaskID *taskID) { + if (taskID == NULL || taskID->async < MIN_ASYNC_ID || taskID->async > MAX_ASYNC_ID || taskID->id <= 0) { return TSDB_CODE_INVALID_PARA; } int32_t ret = 0; + SVAsync *async = vnodeAsyncs[taskID->async]; SVATask *task = NULL; - SVATask task2 = {.taskId = taskId}; + SVATask task2 = { + .taskId = taskID->id, + }; + void (*cancel)(void *) = NULL; + void *arg = NULL; taosThreadMutexLock(&async->mutex); vHashGet(async->taskTable, &task2, (void **)&task); if (task) { if (task->state == EVA_TASK_STATE_WAITTING) { - // remove from queue + cancel = task->cancel; + arg = task->arg; task->next->prev = task->prev; task->prev->next = task->next; vnodeAsyncTaskDone(async, task); @@ -599,14 +648,18 @@ int32_t vnodeACancel(SVAsync *async, int64_t taskId) { taosThreadMutexUnlock(&async->mutex); + if (cancel) { + cancel(arg); + } + return ret; } -int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) { - if (async == NULL || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) { +int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) { + if (asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID || numWorkers <= 0 || numWorkers > VNODE_ASYNC_MAX_WORKERS) { return TSDB_CODE_INVALID_PARA; } - + SVAsync *async = vnodeAsyncs[asyncID]; taosThreadMutexLock(&async->mutex); async->numWorkers = numWorkers; if (async->numIdleWorkers > 0) { @@ -617,11 +670,13 @@ int32_t vnodeAsyncSetWorkers(SVAsync *async, int32_t numWorkers) { return 0; } -int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { - if (async == NULL || channelId == NULL) { +int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) { + if (channelID == NULL || asyncID < MIN_ASYNC_ID || asyncID > MAX_ASYNC_ID) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[asyncID]; + // create channel object SVAChannel *channel = (SVAChannel *)taosMemoryMalloc(sizeof(SVAChannel)); if (channel == NULL) { @@ -637,7 +692,7 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { // register channel taosThreadMutexLock(&async->mutex); - channel->channelId = *channelId = ++async->nextChannelId; + channel->channelId = channelID->id = ++async->nextChannelId; // add to hash table int32_t ret = vHashPut(async->channelTable, channel); @@ -657,16 +712,24 @@ int32_t vnodeAChannelInit(SVAsync *async, int64_t *channelId) { taosThreadMutexUnlock(&async->mutex); + channelID->async = asyncID; return 0; } -int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning) { - if (async == NULL || channelId <= 0) { +int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) { + if (channelID == NULL || channelID->async < MIN_ASYNC_ID || channelID->async > MAX_ASYNC_ID || channelID->id <= 0) { return TSDB_CODE_INVALID_PARA; } + SVAsync *async = vnodeAsyncs[channelID->async]; SVAChannel *channel = NULL; - SVAChannel channel2 = {.channelId = channelId}; + SVAChannel channel2 = { + .channelId = channelID->id, + }; + SArray *cancelArray = taosArrayInit(0, sizeof(SVATaskCancelInfo)); + if (cancelArray == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } taosThreadMutexLock(&async->mutex); @@ -684,6 +747,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning SVATask *task = channel->queue[i].next; task->prev->next = task->next; task->next->prev = task->prev; + if (task->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = task->cancel, + .arg = task->arg, + }); + } vnodeAsyncTaskDone(async, task); } } @@ -693,6 +762,12 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning if (channel->scheduled) { channel->scheduled->prev->next = channel->scheduled->next; channel->scheduled->next->prev = channel->scheduled->prev; + if (channel->scheduled->cancel) { + taosArrayPush(cancelArray, &(SVATaskCancelInfo){ + .cancel = channel->scheduled->cancel, + .arg = channel->scheduled->arg, + }); + } vnodeAsyncTaskDone(async, channel->scheduled); } taosMemoryFree(channel); @@ -713,12 +788,16 @@ int32_t vnodeAChannelDestroy(SVAsync *async, int64_t channelId, bool waitRunning channel->state = EVA_CHANNEL_STATE_CLOSE; } } - } else { - taosThreadMutexUnlock(&async->mutex); - return TSDB_CODE_INVALID_PARA; } taosThreadMutexUnlock(&async->mutex); + for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) { + SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i); + cancel->cancel(cancel->arg); + } + taosArrayDestroy(cancelArray); + channelID->async = 0; + channelID->id = 0; return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 645f2620dc..5acaf2bce4 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -160,6 +160,12 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) { (atExit && (pVnode->inUse->size > 0 || pVnode->pMeta->changed || pVnode->state.applied - pVnode->state.committed > 4096)); } + vTrace("vgId:%d, should commit:%d, disk available:%d, buffer size:%" PRId64 ", node size:%" PRId64 + ", meta changed:%d" + ", state:[%" PRId64 ",%" PRId64 "]", + TD_VID(pVnode), needCommit, diskAvail, pVnode->inUse ? pVnode->inUse->size : 0, + pVnode->inUse ? pVnode->inUse->node.size : 0, pVnode->pMeta->changed, pVnode->state.applied, + pVnode->state.committed); taosThreadMutexUnlock(&pVnode->mutex); return needCommit; } @@ -289,7 +295,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { int64_t lastCommitted = pInfo->info.state.committed; // wait last commit task - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); + vnodeAWait(&pVnode->commitTask); if (syncNodeGetConfig(pVnode->sync, &pVnode->config.syncCfg) != 0) goto _exit; @@ -361,15 +367,14 @@ static void vnodeReturnBufPool(SVnode *pVnode) { taosThreadMutexUnlock(&pVnode->mutex); } -static int32_t vnodeCommitTask(void *arg) { +static int32_t vnodeCommit(void *arg) { int32_t code = 0; SCommitInfo *pInfo = (SCommitInfo *)arg; SVnode *pVnode = pInfo->pVnode; // commit - code = vnodeCommitImpl(pInfo); - if (code) { + if ((code = vnodeCommitImpl(pInfo))) { vFatal("vgId:%d, failed to commit vnode since %s", TD_VID(pVnode), terrstr()); taosMsleep(100); exit(EXIT_FAILURE); @@ -379,37 +384,34 @@ static int32_t vnodeCommitTask(void *arg) { vnodeReturnBufPool(pVnode); _exit: + taosMemoryFree(arg); return code; } -static void vnodeCompleteCommit(void *arg) { taosMemoryFree(arg); } +static void vnodeCommitCancel(void *arg) { taosMemoryFree(arg); } int vnodeAsyncCommit(SVnode *pVnode) { int32_t code = 0; + int32_t lino = 0; SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); if (NULL == pInfo) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code = TSDB_CODE_OUT_OF_MEMORY, lino, _exit); } // prepare to commit code = vnodePrepareCommit(pVnode, pInfo); - if (TSDB_CODE_SUCCESS != code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); // schedule the task - code = vnodeAsyncC(vnodeAsyncHandle[0], pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommitTask, - vnodeCompleteCommit, pInfo, &pVnode->commitTask); + code = + vnodeAsync(&pVnode->commitChannel, EVA_PRIORITY_HIGH, vnodeCommit, vnodeCommitCancel, pInfo, &pVnode->commitTask); + TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - if (NULL != pInfo) { - taosMemoryFree(pInfo); - } - vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), - pVnode->state.commitID); + taosMemoryFree(pInfo); + vError("vgId:%d %s failed at line %d since %s" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code)); } else { vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applyTerm, pVnode->state.applied); @@ -419,7 +421,7 @@ _exit: int vnodeSyncCommit(SVnode *pVnode) { vnodeAsyncCommit(pVnode); - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); + vnodeAWait(&pVnode->commitTask); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeHash.h b/source/dnode/vnode/src/vnd/vnodeHash.h index 0181ca748d..00b3488930 100644 --- a/source/dnode/vnode/src/vnd/vnodeHash.h +++ b/source/dnode/vnode/src/vnd/vnodeHash.h @@ -22,23 +22,6 @@ extern "C" { #endif -typedef struct SVHashTable SVHashTable; - -struct SVHashTable { - uint32_t (*hash)(const void*); - int32_t (*compare)(const void*, const void*); - int32_t numEntries; - uint32_t numBuckets; - struct SVHashEntry** buckets; -}; - -#define vHashNumEntries(ht) ((ht)->numEntries) -int32_t vHashInit(SVHashTable** ht, uint32_t (*hash)(const void*), int32_t (*compare)(const void*, const void*)); -int32_t vHashDestroy(SVHashTable** ht); -int32_t vHashPut(SVHashTable* ht, void* obj); -int32_t vHashGet(SVHashTable* ht, const void* obj, void** retObj); -int32_t vHashDrop(SVHashTable* ht, const void* obj); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 44fcbefba7..3a454c53ef 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -18,8 +18,6 @@ static volatile int32_t VINIT = 0; -SVAsync* vnodeAsyncHandle[2]; - int vnodeInit(int nthreads) { int32_t init; @@ -28,13 +26,9 @@ int vnodeInit(int nthreads) { return 0; } - // vnode-commit - vnodeAsyncInit(&vnodeAsyncHandle[0], "vnode-commit"); - vnodeAsyncSetWorkers(vnodeAsyncHandle[0], nthreads); - - // vnode-merge - vnodeAsyncInit(&vnodeAsyncHandle[1], "vnode-merge"); - vnodeAsyncSetWorkers(vnodeAsyncHandle[1], nthreads); + if (vnodeAsyncOpen(nthreads) != 0) { + return -1; + } if (walInit() < 0) { return -1; @@ -48,8 +42,7 @@ void vnodeCleanup() { if (init == 0) return; // set stop - vnodeAsyncDestroy(&vnodeAsyncHandle[0]); - vnodeAsyncDestroy(&vnodeAsyncHandle[1]); + vnodeAsyncClose(); walCleanUp(); smaCleanUp(); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 425cff7ce9..da8c3a6cad 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -399,7 +399,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - if (vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel) != 0) { + if (vnodeAChannelInit(1, &pVnode->commitChannel) != 0) { vError("vgId:%d, failed to init commit channel", TD_VID(pVnode)); goto _err; } @@ -527,8 +527,8 @@ void vnodePostClose(SVnode *pVnode) { vnodeSyncPostClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { - vnodeAWait(vnodeAsyncHandle[0], pVnode->commitTask); - vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + vnodeAWait(&pVnode->commitTask); + vnodeAChannelDestroy(&pVnode->commitChannel, true); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); tqClose(pVnode->pTq); diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index 5db20b8fc7..6dca7a9a60 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,20 +15,15 @@ #include "vnd.h" -int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; +extern int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now); +extern int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now); - code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); - - if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now); - - return code; +int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now) { + // async retention + return tsdbAsyncRetention(pVnode->pTsdb, now); } -int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now) { - int32_t code = TSDB_CODE_SUCCESS; - - code = tsdbS3Migrate(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); - - return code; +int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now) { + // async migration + return tsdbAsyncS3Migrate(pVnode->pTsdb, now); } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index ea742108aa..611a603c63 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -422,7 +422,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) } // STREAM ============ - vInfo("vgId:%d stream task start", vgId); + vInfo("vgId:%d stream task start to take snapshot", vgId); if (!pReader->streamTaskDone) { if (pReader->pStreamTaskReader == NULL) { code = streamTaskSnapReaderOpen(pReader->pVnode->pTq, pReader->sver, pReader->sver, &pReader->pStreamTaskReader); @@ -622,13 +622,13 @@ extern int32_t tsdbEnableBgTask(STsdb *pTsdb); static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) { tsdbDisableAndCancelAllBgTask(pVnode->pTsdb); vnodeSyncCommit(pVnode); - vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true); + vnodeAChannelDestroy(&pVnode->commitChannel, true); return 0; } static int32_t vnodeEnableBgTask(SVnode *pVnode) { tsdbEnableBgTask(pVnode->pTsdb); - vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel); + vnodeAChannelInit(1, &pVnode->commitChannel); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cb71495e9f..002f04b8a7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -50,7 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token); -static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp); +static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg); @@ -625,6 +625,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg goto _err; } } break; + case TDMT_STREAM_TASK_UPDATE_CHKPT: { + if (tqProcessTaskUpdateCheckpointReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { + goto _err; + } + } break; case TDMT_STREAM_TASK_PAUSE: { if (pVnode->restored && vnodeIsLeader(pVnode) && tqProcessTaskPauseReq(pVnode->pTq, ver, pMsg->pCont, pMsg->contLen) < 0) { @@ -837,12 +842,16 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessTaskScanHistory(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_CHECKPOINT_READY: return tqProcessTaskCheckpointReadyMsg(pVnode->pTq, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: + return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_RETRIEVE_TRIGGER: + return tqProcessTaskRetrieveTriggerReq(pVnode->pTq, pMsg); + case TDMT_STREAM_RETRIEVE_TRIGGER_RSP: + return tqProcessTaskRetrieveTriggerRsp(pVnode->pTq, pMsg); case TDMT_MND_STREAM_HEARTBEAT_RSP: return tqProcessStreamHbRsp(pVnode->pTq, pMsg); case TDMT_MND_STREAM_REQ_CHKPT_RSP: return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); - case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: - return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg); case TDMT_VND_GET_STREAM_PROGRESS: return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg); default: @@ -866,7 +875,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { pMetaRsp->precision = pVnode->config.tsdbCfg.precision; } -extern int32_t vnodeDoRetention(SVnode *pVnode, int64_t now); +extern int32_t vnodeAsyncRetention(SVnode *pVnode, int64_t now); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; @@ -880,13 +889,13 @@ static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int3 vInfo("vgId:%d, trim vnode request will be processed, time:%d", pVnode->config.vgId, trimReq.timestamp); - code = vnodeDoRetention(pVnode, trimReq.timestamp); + code = vnodeAsyncRetention(pVnode, trimReq.timestamp); _exit: return code; } -extern int32_t vnodeDoS3Migrate(SVnode *pVnode, int64_t now); +extern int32_t vnodeAsyncS3Migrate(SVnode *pVnode, int64_t now); static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = 0; @@ -900,7 +909,7 @@ static int32_t vnodeProcessS3MigrateReq(SVnode *pVnode, int64_t ver, void *pReq, vInfo("vgId:%d, s3migrate vnode request will be processed, time:%d", pVnode->config.vgId, s3migrateReq.timestamp); - code = vnodeDoS3Migrate(pVnode, s3migrateReq.timestamp); + code = vnodeAsyncS3Migrate(pVnode, s3migrateReq.timestamp); _exit: return code; @@ -931,13 +940,13 @@ end: return ret; } -static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) { +static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { int32_t code = -1; SMetaReader mr = {0}; SVDropTtlTableReq ttlReq = {0}; SVFetchTtlExpiredTbsRsp rsp = {0}; SEncoder encoder = {0}; - SArray* pNames = NULL; + SArray *pNames = NULL; pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; @@ -950,8 +959,8 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids)); - tb_uid_t suid; - char ctbName[TSDB_TABLE_NAME_LEN]; + tb_uid_t suid; + char ctbName[TSDB_TABLE_NAME_LEN]; SVDropTbReq expiredTb = {.igNotExists = true}; metaReaderDoInit(&mr, pVnode->pMeta, 0); rsp.vgId = TD_VID(pVnode); @@ -965,12 +974,12 @@ static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* } char buf[TSDB_TABLE_NAME_LEN]; for (int32_t i = 0; i < ttlReq.nUids; ++i) { - tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i); + tb_uid_t *uid = taosArrayGet(ttlReq.pTbUids, i); expiredTb.suid = *uid; terrno = metaReaderGetTableEntryByUid(&mr, *uid); if (terrno < 0) goto _end; strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN); - void* p = taosArrayPush(pNames, buf); + void *p = taosArrayPush(pNames, buf); expiredTb.name = p; if (mr.me.type == TSDB_CHILD_TABLE) { expiredTb.suid = mr.me.ctbEntry.suid; @@ -1144,7 +1153,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, if (i < tbNames->size - 1) { taosStringBuilderAppendChar(&sb, ','); } - //taosMemoryFreeClear(*key); + // taosMemoryFreeClear(*key); } size_t len = 0; @@ -1158,12 +1167,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } _exit: - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - pCreateReq = req.pReqs + iReq; - taosMemoryFree(pCreateReq->sql); - taosMemoryFree(pCreateReq->comment); - taosArrayDestroy(pCreateReq->ctb.tagName); - } + tDeleteSVCreateTbBatchReq(&req); taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); tDecoderClear(&decoder); @@ -2241,14 +2245,14 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, return TSDB_CODE_SUCCESS; } -extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); +extern int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { if (!pVnode->restored) { vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver); return 0; } - return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); + return vnodeAsyncCompact(pVnode, ver, pReq, len, pRsp); } static int32_t vnodeProcessConfigChangeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { @@ -2355,8 +2359,6 @@ _OVER: } #ifndef TD_ENTERPRISE -int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { - return 0; -} +int32_t vnodeAsyncCompact(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { return 0; } int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, bool sync) { return 0; } #endif diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 166a230c76..8f28871e3b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -576,20 +576,18 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) if (tsDisableStream) { vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { - vInfo("vgId:%d sync restore finished, start to launch stream task(s)", pVnode->config.vgId); - int32_t numOfTasks = tqStreamTasksGetTotalNum(pMeta); - if (numOfTasks > 0) { - if (pMeta->startInfo.startAllTasks == 1) { - pMeta->startInfo.restartCount += 1; - tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, - pMeta->startInfo.restartCount); - } else { - pMeta->startInfo.startAllTasks = 1; + vInfo("vgId:%d sync restore finished, start to launch stream task(s)", vgId); + if (pMeta->startInfo.startAllTasks == 1) { + pMeta->startInfo.restartCount += 1; + vDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, + pMeta->startInfo.restartCount); + } else { + pMeta->startInfo.startAllTasks = 1; + streamMetaWUnLock(pMeta); - streamMetaWUnLock(pMeta); - tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); - return; - } + tqInfo("vgId:%d stream task already loaded, start them", vgId); + streamTaskSchedTask(&pVnode->msgCb, TD_VID(pVnode), 0, 0, STREAM_EXEC_T_START_ALL_TASKS); + return; } } } else { diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 886ce9705d..18f51df2e9 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -58,7 +58,7 @@ typedef struct STaskStopInfo { typedef struct { STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta + SMqBatchMetaRsp btMetaRsp; // for tmq fetching meta int8_t sourceExcluded; int64_t snapshotVer; SSchemaWrapper* schema; diff --git a/source/libs/executor/src/countwindowoperator.c b/source/libs/executor/src/countwindowoperator.c index 1f38264644..47b24421c8 100644 --- a/source/libs/executor/src/countwindowoperator.c +++ b/source/libs/executor/src/countwindowoperator.c @@ -67,7 +67,12 @@ static void clearWinStateBuff(SCountWindowResult* pBuff) { static SCountWindowResult* getCountWinStateInfo(SCountWindowSupp* pCountSup) { SCountWindowResult* pBuffInfo = taosArrayGet(pCountSup->pWinStates, pCountSup->stateIndex); - pCountSup->stateIndex = (pCountSup->stateIndex + 1) % taosArrayGetSize(pCountSup->pWinStates); + int32_t size = taosArrayGetSize(pCountSup->pWinStates); + // coverity scan + ASSERTS(size > 0, "WinStates is empty"); + if (size > 0) { + pCountSup->stateIndex = (pCountSup->stateIndex + 1) % size; + } return pBuffInfo; } diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 2cba6e3241..29907e6f1f 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -202,6 +202,7 @@ static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) { } eventWindowAggImpl(pOperator, pInfo, pBlock); + doFilter(pRes, pSup->pFilterInfo, NULL); if (pRes->info.rows >= pOperator->resultInfo.threshold) { return pRes; } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 017332cf2d..c527224438 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -95,6 +95,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn goto _error; } + tmemory_barrier(); SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pDataInfo->index); @@ -434,6 +435,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { } } + tmemory_barrier(); pSourceDataInfo->status = EX_SOURCE_DATA_READY; code = tsem_post(&pExchangeInfo->ready); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 390eee2203..77a80d229e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -162,7 +162,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; - } else if (type == STREAM_INPUT__CHECKPOINT) { + } else if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { SPackedData tmp = {.pDataBlock = input}; taosArrayPush(pInfo->pBlockLists, &tmp); pInfo->blockType = STREAM_INPUT__CHECKPOINT; @@ -1079,9 +1079,9 @@ const char* qExtractTbnameFromTask(qTaskInfo_t tinfo) { return pTaskInfo->streamInfo.tbName; } -SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { +SMqBatchMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return &pTaskInfo->streamInfo.metaRsp; + return &pTaskInfo->streamInfo.btMetaRsp; } void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 110aabf9b1..eef8b06ac5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1314,6 +1314,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->tableEndIndex = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; + pTableScanInfo->scanMode = TABLE_SCAN__BLOCK_ORDER; } static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, @@ -1363,6 +1364,12 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal) { + // coverity scan + ASSERTS(pVal != NULL, "pVal should not be NULL"); + if (!pVal) { + qError("failed to compare primary key, since primary key is null"); + return false; + } void* pData = colDataGetData(pCol, rowId); if (IS_VAR_DATA_TYPE(pCol->info.type)) { int32_t colLen = varDataLen(pData); @@ -1469,8 +1476,13 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ } STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info; - qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, pInfo->pUpdateInfo->maxDataVersion); - resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + // coverity scan + ASSERTS(pInfo->pUpdateInfo != NULL, "Failed to set data version, since pInfo->pUpdateInfo is NULL"); + if (pInfo->pUpdateInfo) { + qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRIu64, win.skey, win.ekey, + pInfo->pUpdateInfo->maxDataVersion); + resetTableScanInfo(pInfo->pTableScanOp->info, &win, pInfo->pUpdateInfo->maxDataVersion); + } pInfo->pTableScanOp->status = OP_OPENED; return true; } @@ -2845,8 +2857,8 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { SStreamRawScanInfo* pInfo = pOperator->info; int32_t code = TSDB_CODE_SUCCESS; - pTaskInfo->streamInfo.metaRsp.metaRspLen = 0; // use metaRspLen !=0 to judge if data is meta - pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; + pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = NULL; // use batchMetaReq != NULL to judge if data is meta + pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = NULL; qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -2893,28 +2905,60 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return NULL; } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; - void* data = NULL; - int32_t dataLen = 0; - int16_t type = 0; - int64_t uid = 0; - if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { - qError("tmqsnap getTableInfoFromSnapshot error"); - taosMemoryFreeClear(data); - return NULL; - } + for(int32_t i = 0; i < tmqRowSize; i++) { + void* data = NULL; + int32_t dataLen = 0; + int16_t type = 0; + int64_t uid = 0; + if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) { + qError("tmqsnap getTableInfoFromSnapshot error"); + taosMemoryFreeClear(data); + break; + } - if (!sContext->queryMeta) { // change to get data next poll request - STqOffsetVal offset = {0}; - SValue val = {0}; - tqOffsetResetToData(&offset, 0, INT64_MIN, val); - qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); - } else { - tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); - pTaskInfo->streamInfo.metaRsp.resMsgType = type; - pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; - pTaskInfo->streamInfo.metaRsp.metaRsp = data; - } + if (!sContext->queryMeta) { // change to get data next poll request + STqOffsetVal offset = {0}; + SValue val = {0}; + tqOffsetResetToData(&offset, 0, INT64_MIN, val); + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); + break; + } else { + tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); + SMqMetaRsp tmpMetaRsp = {0}; + tmpMetaRsp.resMsgType = type; + tmpMetaRsp.metaRspLen = dataLen; + tmpMetaRsp.metaRsp = data; + if (!pTaskInfo->streamInfo.btMetaRsp.batchMetaReq) { + pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES); + pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t)); + } + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, code); + if (TSDB_CODE_SUCCESS != code) { + qError("tmqsnap tEncodeMqMetaRsp error"); + taosMemoryFreeClear(data); + break; + } + int32_t tLen = sizeof(SMqRspHead) + len; + void* tBuf = taosMemoryCalloc(1, tLen); + void* metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead)); + SEncoder encoder = {0}; + tEncoderInit(&encoder, metaBuff, len); + code = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp); + if (code < 0) { + qError("tmqsnap tEncodeMqMetaRsp error"); + tEncoderClear(&encoder); + taosMemoryFreeClear(tBuf); + taosMemoryFreeClear(data); + break; + } + taosMemoryFreeClear(data); + taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaReq, &tBuf); + taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaLen, &tLen); + } + } return NULL; } return NULL; diff --git a/source/libs/executor/src/streamcountwindowoperator.c b/source/libs/executor/src/streamcountwindowoperator.c index 050e67e15d..857d048457 100644 --- a/source/libs/executor/src/streamcountwindowoperator.c +++ b/source/libs/executor/src/streamcountwindowoperator.c @@ -677,6 +677,14 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys goto _error; } + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pCountNode->window.watermark, + .calTrigger = pCountNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + .deleteMark = getDeleteMark(&pCountNode->window, 0), + }; + pInfo->primaryTsIndex = ((SColumnNode*)pCountNode->window.pTspk)->slotId; code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, sizeof(COUNT_TYPE), 0, &pTaskInfo->storageAPI.stateStore, pHandle, @@ -687,13 +695,6 @@ SOperatorInfo* createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->streamAggSup.windowCount = pCountNode->windowCount; pInfo->streamAggSup.windowSliding = pCountNode->windowSliding; - pInfo->twAggSup = (STimeWindowAggSupp){ - .waterMark = pCountNode->window.watermark, - .calTrigger = pCountNode->window.triggerType, - .maxTs = INT64_MIN, - .minTs = INT64_MAX, - }; - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->binfo.pRes = pResBlock; diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 64d6244fe7..1809d0cc67 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -364,7 +364,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - code = saveResult(curWin.winInfo, pSeUpdated); + saveResult(curWin.winInfo, pSeUpdated); } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 2da9ed0353..2224942893 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -855,7 +855,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { - pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { @@ -1427,7 +1427,9 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } - removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) { + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + } if (IS_FINAL_INTERVAL_OP(pOperator)) { closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); @@ -2142,7 +2144,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData int32_t pkLen = 0; SColumnInfoData* pPkColDataInfo = NULL; if (hasSrcPrimaryKeyCol(&pInfo->basic)) { - pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex); } for (int32_t i = 0; i < rows;) { @@ -2845,7 +2847,9 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc); - removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); + if (!pInfo->destHasPrimaryKey) { + removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated); + } if (pInfo->isHistoryOp) { getMaxTsWins(pInfo->pUpdated, pInfo->historyWins); } @@ -4131,7 +4135,9 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } pOperator->status = OP_RES_TO_RETURN; - removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + if (!pInfo->destHasPrimaryKey) { + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + } closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) { diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 9a84127be6..cda22fa320 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -194,6 +194,7 @@ static void tRowGetKeyFromColData(int64_t ts, SColumnInfoData* pPkCol, int32_t r } } +// only the timestamp is needed to complete the duplicated timestamp check. static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumnInfoData* pTsCol, SColumnInfoData* pPkCol, int32_t curIndex, int32_t rows) { int64_t currentTs = *(int64_t*)colDataGetData(pTsCol, curIndex); @@ -511,7 +512,6 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB } pInfo->isPrevRowSet = false; - return TSDB_CODE_SUCCESS; } @@ -825,8 +825,12 @@ static void genInterpAfterDataBlock(STimeSliceOperatorInfo* pSliceInfo, SOperato SSDataBlock* pResBlock = pSliceInfo->pRes; SInterval* pInterval = &pSliceInfo->interval; - while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT && - pSliceInfo->fillType != TSDB_FILL_LINEAR) { + if (pSliceInfo->fillType == TSDB_FILL_NEXT || pSliceInfo->fillType == TSDB_FILL_LINEAR || + pSliceInfo->pPrevGroupKey == NULL) { + return; + } + + while (pSliceInfo->current <= pSliceInfo->win.ekey) { genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, NULL, index, false); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -1068,6 +1072,8 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); +// int32_t code = initKeeperInfo(pSliceInfo, pBlock, &pOperator->exprSupp); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index daac98bbfc..8ab388830f 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -989,13 +989,15 @@ void tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupleHa colDataSetNULL(pColInfo, pBlock->info.rows); } } - if (bFreeRow) { - taosMemoryFree(buf); - } + if (*(int32_t*)pStart != pStart - buf) { qError("table merge scan row buf deserialization. length error %d != %d ", *(int32_t*)pStart, (int32_t)(pStart - buf)); - }; + } + + if (bFreeRow) { + taosMemoryFree(buf); + } pBlock->info.dataLoad = 1; pBlock->info.scanFlag = ((SDataBlockInfo*)tsortGetBlockInfo(pTupleHandle))->scanFlag; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index e2ea6c1374..54745951cc 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1242,14 +1242,9 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) { } static void udfdPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("buildInfo: %s\n", buildinfo); + printf("udfd version: %s compatible_version: %s\n", version, compatible_version); + printf("git: %s\n", gitinfo); + printf("build: %s\n", buildinfo); } static int32_t udfdInitLog() { @@ -1424,7 +1419,7 @@ int main(int argc, char *argv[]) { printf("failed to start since init log error\n"); } - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) { + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { fnError("failed to start since read config error"); return -2; } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index eef6cdc3d3..aa8b88b738 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -127,7 +127,7 @@ int aggregateFuncTest() { int main(int argc, char *argv[]) { parseArgs(argc, argv); initLog(); - if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) { + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { fnError("failed to start since read config error"); return -1; } diff --git a/source/libs/monitorfw/inc/taos_linked_list_i.h b/source/libs/monitorfw/inc/taos_linked_list_i.h index ed11a76427..015f8a57ad 100644 --- a/source/libs/monitorfw/inc/taos_linked_list_i.h +++ b/source/libs/monitorfw/inc/taos_linked_list_i.h @@ -47,7 +47,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item); /** * @brief API PRIVATE Pop the first item off of the list */ -void *taos_linked_list_pop(taos_linked_list_t *self); +//void *taos_linked_list_pop(taos_linked_list_t *self); /** * @brief API PRIVATE Returns the item at the head of the list or NULL if not present diff --git a/source/libs/monitorfw/src/taos_collector.c b/source/libs/monitorfw/src/taos_collector.c index 17d324462c..414c02121d 100644 --- a/source/libs/monitorfw/src/taos_collector.c +++ b/source/libs/monitorfw/src/taos_collector.c @@ -33,6 +33,8 @@ taos_map_t *taos_collector_default_collect(taos_collector_t *self) { return self taos_collector_t *taos_collector_new(const char *name) { int r = 0; taos_collector_t *self = (taos_collector_t *)taos_malloc(sizeof(taos_collector_t)); + if (self == NULL) return NULL; + memset(self, 0, sizeof(taos_collector_t)); self->name = taos_strdup(name); self->metrics = taos_map_new(); if (self->metrics == NULL) { @@ -66,9 +68,11 @@ int taos_collector_destroy(taos_collector_t *self) { if (r) ret = r; self->metrics = NULL; - r = taos_string_builder_destroy(self->string_builder); - if (r) ret = r; - self->string_builder = NULL; + if(self->string_builder != NULL){ + r = taos_string_builder_destroy(self->string_builder); + if (r) ret = r; + self->string_builder = NULL; + } taos_free((char *)self->name); self->name = NULL; diff --git a/source/libs/monitorfw/src/taos_collector_registry.c b/source/libs/monitorfw/src/taos_collector_registry.c index c3ed0112c5..d4838ef301 100644 --- a/source/libs/monitorfw/src/taos_collector_registry.c +++ b/source/libs/monitorfw/src/taos_collector_registry.c @@ -59,6 +59,7 @@ taos_collector_registry_t *taos_collector_registry_new(const char *name) { r = pthread_rwlock_init(self->lock, NULL); if (r) { TAOS_LOG("failed to initialize rwlock"); + taos_free(self); return NULL; } return self; @@ -301,6 +302,9 @@ const char *taos_collector_registry_bridge_new(taos_collector_registry_t *self, _OVER: tjsonDelete(pJson); + if(tmp_builder != NULL){ + taos_string_builder_destroy(tmp_builder); + } return NULL; } diff --git a/source/libs/monitorfw/src/taos_linked_list.c b/source/libs/monitorfw/src/taos_linked_list.c index 675400a6fa..2becb08a07 100644 --- a/source/libs/monitorfw/src/taos_linked_list.c +++ b/source/libs/monitorfw/src/taos_linked_list.c @@ -117,6 +117,7 @@ int taos_linked_list_push(taos_linked_list_t *self, void *item) { return 0; } +/* void *taos_linked_list_pop(taos_linked_list_t *self) { TAOS_ASSERT(self != NULL); if (self == NULL) return NULL; @@ -141,6 +142,7 @@ void *taos_linked_list_pop(taos_linked_list_t *self) { } return item; } +*/ int taos_linked_list_remove(taos_linked_list_t *self, void *item) { TAOS_ASSERT(self != NULL); diff --git a/source/libs/monitorfw/src/taos_map.c b/source/libs/monitorfw/src/taos_map.c index 8f0b83884e..ffb7d000fc 100644 --- a/source/libs/monitorfw/src/taos_map.c +++ b/source/libs/monitorfw/src/taos_map.c @@ -90,7 +90,7 @@ taos_map_t *taos_map_new() { return NULL; } - self->addrs = taos_malloc(sizeof(taos_linked_list_t) * self->max_size); + self->addrs = taos_malloc(sizeof(taos_linked_list_t*) * self->max_size); self->free_value_fn = destroy_map_node_value_no_op; for (int i = 0; i < self->max_size; i++) { @@ -273,7 +273,7 @@ int taos_map_ensure_space(taos_map_t *self) { if (r) return r; // Create a new array of addrs - taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t) * new_max); + taos_linked_list_t **new_addrs = taos_malloc(sizeof(taos_linked_list_t*) * new_max); // Initialize the new array for (int i = 0; i < new_max; i++) { diff --git a/source/libs/monitorfw/src/taos_metric.c b/source/libs/monitorfw/src/taos_metric.c index 4e9af35f34..5cecbc927f 100644 --- a/source/libs/monitorfw/src/taos_metric.c +++ b/source/libs/monitorfw/src/taos_metric.c @@ -33,6 +33,8 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, size_t label_key_count, const char **label_keys) { int r = 0; taos_metric_t *self = (taos_metric_t *)taos_malloc(sizeof(taos_metric_t)); + if (self == NULL) return NULL; + memset(self, 0, sizeof(taos_metric_t)); self->type = metric_type; int len = strlen(name) + 1; self->name = taos_malloc(len); @@ -79,6 +81,7 @@ taos_metric_t *taos_metric_new(taos_metric_type_t metric_type, const char *name, r = pthread_rwlock_init(self->rwlock, NULL); if (r) { TAOS_LOG(TAOS_PTHREAD_RWLOCK_INIT_ERROR); + taos_free(self); return NULL; } return self; @@ -91,9 +94,11 @@ int taos_metric_destroy(taos_metric_t *self) { int r = 0; int ret = 0; - r = taos_map_destroy(self->samples); - self->samples = NULL; - if (r) ret = r; + if(self->samples != NULL){ + r = taos_map_destroy(self->samples); + self->samples = NULL; + if (r) ret = r; + } r = taos_metric_formatter_destroy(self->formatter); self->formatter = NULL; @@ -152,8 +157,7 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const return NULL; // Get l_value - r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, - label_values); + r = taos_metric_formatter_load_l_value(self->formatter, self->name, NULL, self->label_key_count, self->label_keys, label_values); if (r) { TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); } @@ -170,10 +174,12 @@ taos_metric_sample_t *taos_metric_sample_from_labels(taos_metric_t *self, const sample = taos_metric_sample_new(self->type, l_value, 0.0); r = taos_map_set(self->samples, l_value, sample); if (r) { + taos_free((void *)l_value); TAOS_METRIC_SAMPLE_FROM_LABELS_HANDLE_UNLOCK(); } } - pthread_rwlock_unlock(self->rwlock); + r = pthread_rwlock_unlock(self->rwlock); + if (r) TAOS_LOG(TAOS_PTHREAD_RWLOCK_UNLOCK_ERROR); taos_free((void *)l_value); return sample; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 1a51620856..84d3f734fe 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -845,6 +845,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(isEmptyResult); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); COPY_SCALAR_FIELD(timeLineCurMode); COPY_SCALAR_FIELD(hasAggFuncs); COPY_SCALAR_FIELD(hasRepeatScanFuncs); @@ -862,6 +863,8 @@ static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) { COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); COPY_SCALAR_FIELD(timeLineResMode); + COPY_SCALAR_FIELD(timeLineFromOrderBy); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index cbffcde875..beedffc4f2 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -240,6 +240,26 @@ int32_t nodesCreateAllocator(int64_t queryId, int32_t chunkSize, int64_t* pAlloc return code; } +int32_t nodesSimAcquireAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + SNodeAllocator* pAllocator = taosAcquireRef(g_allocatorReqRefPool, allocatorId); + if (NULL == pAllocator) { + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t nodesSimReleaseAllocator(int64_t allocatorId) { + if (allocatorId <= 0) { + return TSDB_CODE_SUCCESS; + } + + return taosReleaseRef(g_allocatorReqRefPool, allocatorId); +} + int32_t nodesAcquireAllocator(int64_t allocatorId) { if (allocatorId <= 0) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index e8596c4d19..90c3753ed0 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -118,44 +118,46 @@ SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode); SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName); -SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); -SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode); -SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight); +SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pNode); +SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode* pLeft, + SNode* pRight); SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral); -SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); -SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias); -SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); -SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); -SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); -SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); -SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); -SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); -SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); -SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); -SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); -SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); -SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, SNode* pJoinCond); -SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName); -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); -SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); -SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); -SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); -SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken); -SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, - SNode* pFill); -SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset); -SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); -SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); -SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); -SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); -SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); -SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); +SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt); +SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); +SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); +SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, + SNode* pJoinCond); +SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); +SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, + SNode* pFill); +SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); +SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode); +SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd); +SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint); +SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen); +SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse); SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); @@ -170,7 +172,8 @@ SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery); SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill); SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit); SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset); -SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint); +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, + SNodeList* pHint); SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); @@ -205,8 +208,12 @@ SNode* createDropSuperTableStmt(SAstCreateContext* pCxt, bool ignoreNotExists, S SNode* createAlterTableModifyOptions(SAstCreateContext* pCxt, SNode* pRealTable, SNode* pOptions); SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, SDataType dataType); -SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, - SNode* pOptions); + +SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, + SToken* pColName, SDataType dataType, SNode* pOptions); + +SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, + SToken* pColName, SNode* pOptions); SNode* createAlterTableDropCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName); SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pOldColName, SToken* pNewColName); @@ -217,7 +224,8 @@ SNode* setShowKind(SAstCreateContext* pCxt, SNode* pStmt, EShowKind showKind); SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type); SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName, EOperatorType tableCondType); -SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, EOperatorType tableCondType); +SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, SNode* pTbName, + EOperatorType tableCondType); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 60e33c6f33..a5efdbdf91 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -353,7 +353,7 @@ cmd ::= ALTER STABLE alter_table_clause(A). alter_table_clause(A) ::= full_table_name(B) alter_table_options(C). { A = createAlterTableModifyOptions(pCxt, B, C); } alter_table_clause(A) ::= - full_table_name(B) ADD COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D); } + full_table_name(B) ADD COLUMN column_name(C) type_name(D) column_options(E). { A = createAlterTableAddModifyColOptions2(pCxt, B, TSDB_ALTER_TABLE_ADD_COLUMN, &C, D, E); } alter_table_clause(A) ::= full_table_name(B) DROP COLUMN column_name(C). { A = createAlterTableDropCol(pCxt, B, TSDB_ALTER_TABLE_DROP_COLUMN, &C); } alter_table_clause(A) ::= full_table_name(B) MODIFY COLUMN column_name(C) type_name(D). { A = createAlterTableAddModifyCol(pCxt, B, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &C, D); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 9ae9dfffd7..0e31672b24 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -167,7 +167,7 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb) } } else { trimEscape(pDbName); - if (pDbName->n >= TSDB_DB_NAME_LEN) { + if (pDbName->n >= TSDB_DB_NAME_LEN || pDbName->n == 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pDbName->z); } } @@ -176,7 +176,8 @@ static bool checkDbName(SAstCreateContext* pCxt, SToken* pDbName, bool demandDb) static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { trimEscape(pTableName); - if (NULL != pTableName && pTableName->n >= TSDB_TABLE_NAME_LEN) { + if (NULL != pTableName && pTableName->type != TK_NK_NIL && + (pTableName->n >= TSDB_TABLE_NAME_LEN || pTableName->n == 0)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTableName->z); return false; } @@ -185,7 +186,8 @@ static bool checkTableName(SAstCreateContext* pCxt, SToken* pTableName) { static bool checkColumnName(SAstCreateContext* pCxt, SToken* pColumnName) { trimEscape(pColumnName); - if (NULL != pColumnName && pColumnName->n >= TSDB_COL_NAME_LEN) { + if (NULL != pColumnName && pColumnName->type != TK_NK_NIL && + (pColumnName->n >= TSDB_COL_NAME_LEN || pColumnName->n == 0)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pColumnName->z); return false; } @@ -203,7 +205,7 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) { static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) { trimEscape(pTopicName); - if (pTopicName->n >= TSDB_TOPIC_NAME_LEN) { + if (pTopicName->n >= TSDB_TOPIC_NAME_LEN || pTopicName->n == 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTopicName->z); return false; } @@ -221,7 +223,7 @@ static bool checkCGroupName(SAstCreateContext* pCxt, SToken* pCGroup) { static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) { trimEscape(pViewName); - if (pViewName->n >= TSDB_VIEW_NAME_LEN) { + if (pViewName->n >= TSDB_VIEW_NAME_LEN || pViewName->n == 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pViewName->z); return false; } @@ -230,7 +232,7 @@ static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) { static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { trimEscape(pStreamName); - if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { + if (pStreamName->n >= TSDB_STREAM_NAME_LEN || pStreamName->n == 0) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pStreamName->z); return false; } @@ -252,6 +254,8 @@ static bool checkTsmaName(SAstCreateContext* pCxt, SToken* pTsmaToken) { pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; } else if (pTsmaToken->n >= TSDB_TABLE_NAME_LEN - strlen(TSMA_RES_STB_POSTFIX)) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSMA_NAME_TOO_LONG); + } else if (pTsmaToken->n == 0) { + pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTsmaToken->z); } return pCxt->errCode == TSDB_CODE_SUCCESS; } @@ -1891,6 +1895,38 @@ SNode* createAlterTableAddModifyCol(SAstCreateContext* pCxt, SNode* pRealTable, pStmt->dataType = dataType; return createAlterTableStmtFinalize(pRealTable, pStmt); } +SNode* createAlterTableAddModifyColOptions2(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, + SToken* pColName, SDataType dataType, SNode* pOptions) { + CHECK_PARSER_STATUS(pCxt); + if (!checkColumnName(pCxt, pColName)) { + return NULL; + } + + SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->alterType = alterType; + COPY_STRING_FORM_ID_TOKEN(pStmt->colName, pColName); + pStmt->dataType = dataType; + pStmt->pColOptions = (SColumnOptions*)pOptions; + + if (pOptions != NULL) { + SColumnOptions* pOption = (SColumnOptions*)pOptions; + if (pOption->bPrimaryKey == false && pOption->commentNull == true) { + if (strlen(pOption->compress) != 0 || strlen(pOption->compressLevel) || strlen(pOption->encode) != 0) { + pStmt->alterType = TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION; + } else { + // pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + // "not support alter column with option except compress"); + // return NULL; + } + } else { + pCxt->errCode = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "not support alter column with option except compress"); + return NULL; + } + } + return createAlterTableStmtFinalize(pRealTable, pStmt); +} SNode* createAlterTableAddModifyColOptions(SAstCreateContext* pCxt, SNode* pRealTable, int8_t alterType, SToken* pColName, SNode* pOptions) { @@ -1935,6 +1971,7 @@ SNode* createAlterTableRenameCol(SAstCreateContext* pCxt, SNode* pRealTable, int SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken* pTagName, SNode* pVal) { CHECK_PARSER_STATUS(pCxt); if (!checkColumnName(pCxt, pTagName)) { + nodesDestroyNode(pVal); return NULL; } SAlterTableStmt* pStmt = (SAlterTableStmt*)nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); @@ -1946,6 +1983,7 @@ SNode* createAlterTableSetTag(SAstCreateContext* pCxt, SNode* pRealTable, SToken } SNode* setAlterSuperTableType(SNode* pStmt) { + if (!pStmt) return NULL; setNodeType(pStmt, QUERY_NODE_ALTER_SUPER_TABLE_STMT); return pStmt; } @@ -2436,6 +2474,7 @@ SNode* createRestoreComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, c SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) { CHECK_PARSER_STATUS(pCxt); if (!checkTopicName(pCxt, pTopicName)) { + nodesDestroyNode(pQuery); return NULL; } SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); @@ -2721,6 +2760,8 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); if (!checkStreamName(pCxt, pStreamName)) { + nodesDestroyNode(pQuery); + nodesDestroyNode(pOptions); return NULL; } SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); @@ -2809,9 +2850,10 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI return (SNode*)pStmt; } -SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){ +SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName) { CHECK_PARSER_STATUS(pCxt); - SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT); + SBalanceVgroupLeaderStmt* pStmt = + (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT); CHECK_OUT_OF_MEM(pStmt); if (NULL != pDbName) { COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 370514e1fc..22f274b21c 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2393,7 +2393,7 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif // no data in the sql string anymore. if (0 == pTbName->n) { if (0 != pTbName->type && '\0' != pStmt->pSql[0]) { - return buildSyntaxErrMsg(&pCxt->msg, "invalid charactor in SQL", pTbName->z); + return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pTbName->z); } if (0 == pStmt->totalRowsNum && (!TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT))) { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5ad9feddbc..7c43113ad2 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -103,6 +103,9 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c if (pTableName->n >= TSDB_TABLE_NAME_LEN) { return buildInvalidOperationMsg(pMsgBuf, msg1); } + if (pTableName->n == 0) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "invalid table name"); + } char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); @@ -111,6 +114,8 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c if (dbName == NULL) { return buildInvalidOperationMsg(pMsgBuf, msg3); } + if (name[0] == '\0') + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4); code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); if (code != TSDB_CODE_SUCCESS) { @@ -732,6 +737,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } } } else { + bool hasTs = false; for (int i = 0; i < numFields; i++) { for (int j = 0; j < boundInfo->numOfBound; j++) { SSchema* pColSchema = &pSchema[j]; @@ -743,6 +749,10 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } + if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + hasTs = true; + } + int8_t* offset = pStart; if (IS_VAR_DATA_TYPE(pColSchema->type)) { pStart += numOfRows * sizeof(int32_t); @@ -768,6 +778,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } } + if(!hasTs){ + if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } + for (int c = 0; c < boundInfo->numOfBound; ++c) { if (boundInfo->pColIndex[c] != -1) { SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 79b37c2950..1c4dbaa9e1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -959,7 +959,8 @@ static bool isTimeLineQuery(SNode* pStmt) { return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineCurMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineCurMode); } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { - return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; + return (TIME_LINE_MULTI == ((SSetOperator*)pStmt)->timeLineResMode) || + (TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode); } else { return false; } @@ -1000,18 +1001,65 @@ static bool isBlockTimeLineAlignedQuery(SNode* pStmt) { return false; } +SNodeList* buildPartitionListFromOrderList(SNodeList* pOrderList, int32_t nodesNum) { + SNodeList* pPartitionList = NULL; + SNode* pNode = NULL; + if (pOrderList->length <= nodesNum) { + return NULL; + } + + pNode = nodesListGetNode(pOrderList, nodesNum); + SOrderByExprNode* pOrder = (SOrderByExprNode*)pNode; + if (!isPrimaryKeyImpl(pOrder->pExpr)) { + return NULL; + } + + for (int32_t i = 0; i < nodesNum; ++i) { + pNode = nodesListGetNode(pOrderList, i); + pOrder = (SOrderByExprNode*)pNode; + nodesListMakeStrictAppend(&pPartitionList, nodesCloneNode(pOrder->pExpr)); + } + + return pPartitionList; +} + static bool isTimeLineAlignedQuery(SNode* pStmt) { SSelectStmt* pSelect = (SSelectStmt*)pStmt; if (!isTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { return false; } - if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { - return false; + if (QUERY_NODE_SELECT_STMT == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList) { + if (!pSub->timeLineFromOrderBy && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { + return true; + } + if (pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = + buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } + } } - SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery; - if (pSelect->pPartitionByList && nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { - return true; + if (QUERY_NODE_SET_OPERATOR == nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + SSetOperator* pSub = (SSetOperator*)((STempTableNode*)pSelect->pFromTable)->pSubquery; + if (pSelect->pPartitionByList && pSub->timeLineFromOrderBy && pSub->pOrderByList->length > 1) { + SNodeList* pPartitionList = + buildPartitionListFromOrderList(pSub->pOrderByList, pSelect->pPartitionByList->length); + bool match = nodesListMatch(pSelect->pPartitionByList, pPartitionList); + nodesDestroyList(pPartitionList); + + if (match) { + return true; + } + } } + return false; } @@ -6025,9 +6073,19 @@ static void resetResultTimeline(SSelectStmt* pSelect) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder)) || (QUERY_NODE_TEMP_TABLE != nodeType(pSelect->pFromTable) && isPrimaryKeyImpl(pOrder))) { pSelect->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSelect->timeLineResMode = TIME_LINE_NONE; + return; + } else if (pSelect->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSelect->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSelect->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSelect->timeLineResMode = TIME_LINE_MULTI; + pSelect->timeLineFromOrderBy = true; + return; + } + } } + + pSelect->timeLineResMode = TIME_LINE_NONE; } static int32_t replaceOrderByAliasForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { @@ -6180,16 +6238,13 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); - if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight)) { - SColumnNode* pLCol = (SColumnNode*)pLeft; - SColumnNode* pRCol = (SColumnNode*)pRight; + bool isLeftPrimTs = isPrimaryKeyImpl(pLeft); + bool isRightPrimTs = isPrimaryKeyImpl(pRight); + + if (isLeftPrimTs && isRightPrimTs) { SColumnNode* pFCol = (SColumnNode*)pProj; - if (pLCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID && pRCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - if (pLCol->isPrimTs && pRCol->isPrimTs) { - pFCol->isPrimTs = true; - } - } + pFCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + pFCol->isPrimTs = true; } if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -6225,9 +6280,19 @@ static int32_t translateSetOperOrderBy(STranslateContext* pCxt, SSetOperator* pS SNode* pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, 0))->pExpr; if (isPrimaryKeyImpl(pOrder)) { pSetOperator->timeLineResMode = TIME_LINE_GLOBAL; - } else { - pSetOperator->timeLineResMode = TIME_LINE_NONE; + return code; + } else if (pSetOperator->pOrderByList->length > 1) { + for (int32_t i = 1; i < pSetOperator->pOrderByList->length; ++i) { + pOrder = ((SOrderByExprNode*)nodesListGetNode(pSetOperator->pOrderByList, i))->pExpr; + if (isPrimaryKeyImpl(pOrder)) { + pSetOperator->timeLineResMode = TIME_LINE_MULTI; + pSetOperator->timeLineFromOrderBy = true; + return code; + } + } } + + pSetOperator->timeLineResMode = TIME_LINE_NONE; } return code; } @@ -6706,6 +6771,15 @@ static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int return TSDB_CODE_SUCCESS; } +static int32_t checkDbEnumOption3(STranslateContext* pCxt, const char* pName, int32_t val, int32_t v1, int32_t v2, + int32_t v3) { + if (val >= 0 && val != v1 && val != v2 && val != v3) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option %s: %d, only %d, %d, %d allowed", pName, val, v1, v2, v3); + } + return TSDB_CODE_SUCCESS; +} + static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions, int8_t precision) { if (NULL == pRetentions) { return TSDB_CODE_SUCCESS; @@ -6849,7 +6923,12 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa if ((pOptions->replica == 2) ^ (pOptions->withArbitrator == TSDB_MAX_DB_WITH_ARBITRATOR)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid database option, with_arbitrator should be used with replica 2"); + "Invalid option, with_arbitrator should be used with replica 2"); + } + + if (pOptions->replica > 1 && pOptions->walLevel == 0) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option, wal_level 0 should be used with replica 1"); } return TSDB_CODE_SUCCESS; @@ -6913,7 +6992,8 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbStrictOption(pCxt, pOptions); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbEnumOption(pCxt, "walLevel", pOptions->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_MAX_WAL_LEVEL); + code = checkDbEnumOption3(pCxt, "walLevel", pOptions->walLevel, TSDB_MIN_WAL_LEVEL, TSDB_DEFAULT_WAL_LEVEL, + TSDB_MAX_WAL_LEVEL); } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "vgroups", pOptions->numOfVgroups, TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); @@ -7187,6 +7267,15 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, } static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt) { + if (pStmt->pOptions->walLevel == 0) { + SDbCfgInfo dbCfg = {0}; + int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + if (TSDB_CODE_SUCCESS == code && dbCfg.replications > 1) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, + "Invalid option, wal_level 0 should be used with replica 1"); + } + } + int32_t code = checkDatabaseOptions(pCxt, pStmt->dbName, pStmt->pOptions); if (TSDB_CODE_SUCCESS != code) { return code; @@ -8092,9 +8181,9 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* } switch (pStmt->alterType) { + case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_DROP_TAG: - case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: { @@ -8129,6 +8218,31 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* taosArrayPush(pAlterReq->pFields, &field); break; } + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: { + taosArrayDestroy(pAlterReq->pFields); + + pAlterReq->pFields = taosArrayInit(1, sizeof(SFieldWithOptions)); + SFieldWithOptions field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; + // TAOS_FIELD field = {.type = pStmt->dataType.type, .bytes = calcTypeBytes(pStmt->dataType)}; + strcpy(field.name, pStmt->colName); + if (pStmt->pColOptions != NULL) { + if (!checkColumnEncodeOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->encode)) + return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; + if (!checkColumnCompressOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compress)) + return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR; + if (!checkColumnLevelOrSetDefault(pStmt->dataType.type, pStmt->pColOptions->compressLevel)) + return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; + int32_t code = setColCompressByOption(pStmt->dataType.type, columnEncodeVal(pStmt->pColOptions->encode), + columnCompressVal(pStmt->pColOptions->compress), + columnLevelVal(pStmt->pColOptions->compressLevel), false, + (uint32_t*)&field.compress); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + taosArrayPush(pAlterReq->pFields, &field); + break; + } default: break; } @@ -10578,7 +10692,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) { SBalanceVgroupLeaderReq req = {0}; req.vgId = pStmt->vgId; - if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName); + if (pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName); int32_t code = buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req); tFreeSBalanceVgroupLeaderReq(&req); @@ -10993,7 +11107,8 @@ static int32_t buildCreateTSMAReq(STranslateContext* pCxt, SCreateTSMAStmt* pStm } if (TSDB_CODE_SUCCESS == code) { - pReq->deleteMark = convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision); + pReq->deleteMark = + convertTimePrecision(tsmaDataDeleteMark, TSDB_TIME_PRECISION_MILLI, pTableMeta->tableInfo.precision); code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen); } @@ -12793,6 +12908,11 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); } + // only super and normal support + if (pStmt->pColOptions != NULL && TSDB_CHILD_TABLE == pTableMeta->tableType) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); + } + pReq->colName = taosStrdup(pStmt->colName); if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; @@ -12801,6 +12921,17 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S pReq->type = pStmt->dataType.type; pReq->flags = COL_SMA_ON; pReq->bytes = calcTypeBytes(pStmt->dataType); + if (pStmt->pColOptions != NULL) { + if (!checkColumnEncodeOrSetDefault(pReq->type, pStmt->pColOptions->encode)) return TSDB_CODE_TSC_ENCODE_PARAM_ERROR; + if (!checkColumnCompressOrSetDefault(pReq->type, pStmt->pColOptions->compress)) + return TSDB_CODE_TSC_COMPRESS_PARAM_ERROR; + if (!checkColumnLevelOrSetDefault(pReq->type, pStmt->pColOptions->compressLevel)) + return TSDB_CODE_TSC_COMPRESS_LEVEL_ERROR; + int8_t code = setColCompressByOption(pReq->type, columnEncodeVal(pStmt->pColOptions->encode), + columnCompressVal(pStmt->pColOptions->compress), + columnLevelVal(pStmt->pColOptions->compressLevel), true, &pReq->compress); + } + return TSDB_CODE_SUCCESS; } @@ -12947,6 +13078,7 @@ static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: return buildUpdateTagValReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_ADD_COLUMN: + case TSDB_ALTER_TABLE_ADD_COLUMN_WITH_COMPRESS_OPTION: return buildAddColReq(pCxt, pStmt, pTableMeta, pReq); case TSDB_ALTER_TABLE_DROP_COLUMN: return buildDropColReq(pCxt, pStmt, pTableMeta, pReq); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 52dd135cd3..226696532b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -25,6 +25,8 @@ ** input grammar file: */ /************ Begin %include sections from the grammar ************************/ +#line 11 "sql.y" + #include #include #include @@ -40,6 +42,7 @@ #include "parAst.h" #define YYSTACKDEPTH 0 +#line 46 "sql.c" /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols. ***************** Begin token definitions *************************************/ @@ -517,18 +520,18 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 963 +#define YYNSTATE 965 #define YYNRULE 748 #define YYNRULE_WITH_ACTION 748 #define YYNTOKEN 376 -#define YY_MAX_SHIFT 962 -#define YY_MIN_SHIFTREDUCE 1431 -#define YY_MAX_SHIFTREDUCE 2178 -#define YY_ERROR_ACTION 2179 -#define YY_ACCEPT_ACTION 2180 -#define YY_NO_ACTION 2181 -#define YY_MIN_REDUCE 2182 -#define YY_MAX_REDUCE 2929 +#define YY_MAX_SHIFT 964 +#define YY_MIN_SHIFTREDUCE 1432 +#define YY_MAX_SHIFTREDUCE 2179 +#define YY_ERROR_ACTION 2180 +#define YY_ACCEPT_ACTION 2181 +#define YY_NO_ACTION 2182 +#define YY_MIN_REDUCE 2183 +#define YY_MAX_REDUCE 2930 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -597,329 +600,329 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (3228) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 37, 338, 2538, 2705, 639, 542, 2373, 640, 2225, 190, - /* 10 */ 2680, 315, 47, 45, 2095, 2812, 803, 464, 494, 493, - /* 20 */ 469, 2183, 1916, 2535, 845, 40, 39, 2705, 66, 46, - /* 30 */ 44, 43, 42, 41, 1941, 2004, 1941, 1914, 2684, 2480, - /* 40 */ 838, 2809, 136, 1923, 2723, 135, 134, 133, 132, 131, - /* 50 */ 130, 129, 128, 127, 604, 602, 831, 409, 854, 372, - /* 60 */ 2670, 230, 840, 2368, 417, 33, 1999, 830, 2723, 800, + /* 0 */ 37, 338, 2539, 2706, 641, 544, 2374, 642, 2226, 190, + /* 10 */ 2681, 315, 47, 45, 2096, 2813, 805, 466, 496, 495, + /* 20 */ 471, 2184, 1917, 2536, 847, 40, 39, 2706, 66, 46, + /* 30 */ 44, 43, 42, 41, 1942, 2005, 1942, 1915, 2685, 2481, + /* 40 */ 840, 2810, 136, 1924, 2724, 135, 134, 133, 132, 131, + /* 50 */ 130, 129, 128, 127, 606, 604, 833, 410, 856, 372, + /* 60 */ 2671, 230, 842, 2369, 418, 33, 2000, 832, 2724, 802, /* 70 */ 155, 40, 39, 19, 301, 46, 44, 43, 42, 41, - /* 80 */ 1922, 2269, 800, 155, 2670, 210, 840, 724, 136, 2686, - /* 90 */ 2689, 135, 134, 133, 132, 131, 130, 129, 128, 127, - /* 100 */ 859, 2431, 722, 252, 720, 286, 285, 642, 2704, 2233, - /* 110 */ 959, 2743, 2812, 15, 1942, 119, 2706, 844, 2708, 2709, - /* 120 */ 839, 647, 859, 103, 640, 2225, 197, 199, 854, 2797, - /* 130 */ 644, 2139, 2704, 465, 2793, 2743, 641, 62, 2808, 405, - /* 140 */ 2706, 844, 2708, 2709, 839, 837, 859, 823, 2762, 2006, - /* 150 */ 2007, 50, 437, 218, 2511, 753, 932, 931, 930, 929, - /* 160 */ 497, 2844, 928, 927, 160, 922, 921, 920, 919, 918, - /* 170 */ 917, 916, 159, 910, 909, 908, 496, 495, 905, 904, - /* 180 */ 903, 196, 195, 902, 492, 901, 900, 899, 1977, 1987, - /* 190 */ 802, 185, 2805, 2806, 778, 153, 2810, 290, 2005, 2008, - /* 200 */ 800, 155, 1762, 1763, 192, 2805, 799, 579, 147, 798, - /* 210 */ 788, 2663, 578, 1917, 485, 1915, 2895, 9, 2895, 2099, - /* 220 */ 577, 1926, 40, 39, 737, 1941, 46, 44, 43, 42, - /* 230 */ 41, 2360, 2705, 2205, 787, 217, 787, 217, 122, 2896, - /* 240 */ 789, 2896, 789, 1689, 1690, 841, 2388, 487, 2386, 1920, - /* 250 */ 1921, 1974, 777, 1976, 1979, 1980, 1981, 1982, 1983, 1984, - /* 260 */ 1985, 1986, 836, 857, 856, 1998, 2000, 2001, 2002, 2003, - /* 270 */ 2, 47, 45, 2723, 915, 474, 414, 2344, 1939, 469, - /* 280 */ 2723, 1916, 1843, 1844, 2900, 586, 859, 429, 607, 2670, - /* 290 */ 2670, 840, 2895, 606, 2004, 60, 1914, 40, 39, 453, - /* 300 */ 2585, 46, 44, 43, 42, 41, 751, 2014, 734, 1945, - /* 310 */ 565, 2899, 608, 1941, 197, 2896, 2898, 415, 567, 1498, - /* 320 */ 322, 1497, 125, 2805, 2806, 1999, 153, 2810, 180, 545, - /* 330 */ 855, 2384, 19, 662, 322, 2033, 151, 2704, 2387, 1922, - /* 340 */ 2743, 2070, 2512, 773, 119, 2706, 844, 2708, 2709, 839, - /* 350 */ 221, 859, 1471, 776, 157, 1499, 166, 2768, 2797, 2182, - /* 360 */ 40, 39, 465, 2793, 46, 44, 43, 42, 41, 959, - /* 370 */ 438, 1478, 15, 896, 172, 171, 893, 892, 891, 169, - /* 380 */ 1945, 553, 1941, 145, 144, 143, 142, 141, 140, 139, - /* 390 */ 138, 137, 655, 1978, 1473, 1476, 1477, 535, 62, 40, - /* 400 */ 39, 2034, 534, 46, 44, 43, 42, 41, 2006, 2007, - /* 410 */ 659, 2518, 2497, 658, 594, 593, 592, 591, 590, 585, - /* 420 */ 584, 583, 582, 421, 29, 1974, 2070, 572, 571, 570, - /* 430 */ 569, 568, 562, 561, 560, 254, 555, 554, 436, 642, - /* 440 */ 854, 2233, 546, 1750, 1751, 329, 330, 1977, 1987, 1769, - /* 450 */ 328, 754, 779, 774, 767, 763, 1975, 2005, 2008, 2895, - /* 460 */ 2817, 2067, 2068, 2069, 2817, 2817, 2817, 2817, 2817, 2525, - /* 470 */ 2375, 660, 1917, 519, 1915, 62, 898, 2901, 217, 213, - /* 480 */ 738, 1978, 2896, 789, 855, 2384, 855, 2384, 36, 467, - /* 490 */ 2028, 2029, 2030, 2031, 2032, 2036, 2037, 2038, 2039, 2072, - /* 500 */ 2073, 2074, 2075, 2076, 146, 2140, 55, 376, 1920, 1921, - /* 510 */ 1974, 685, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, - /* 520 */ 1986, 836, 857, 856, 1998, 2000, 2001, 2002, 2003, 2, - /* 530 */ 12, 47, 45, 50, 474, 1925, 2705, 2538, 181, 469, - /* 540 */ 1946, 1916, 116, 389, 1975, 859, 2067, 2068, 2069, 803, - /* 550 */ 208, 113, 471, 598, 2004, 1589, 1914, 2170, 2535, 845, - /* 560 */ 387, 76, 649, 2577, 75, 2705, 40, 39, 452, 2585, - /* 570 */ 46, 44, 43, 42, 41, 416, 735, 2723, 841, 2070, - /* 580 */ 2235, 322, 1942, 1922, 2680, 1999, 2900, 250, 621, 619, - /* 590 */ 616, 614, 19, 2670, 2895, 840, 40, 39, 1591, 1922, - /* 600 */ 46, 44, 43, 42, 41, 2490, 2723, 706, 705, 704, - /* 610 */ 523, 1946, 2684, 2899, 696, 152, 700, 2896, 2897, 241, - /* 620 */ 699, 2538, 2670, 656, 840, 698, 703, 446, 445, 959, - /* 630 */ 912, 697, 15, 62, 98, 444, 693, 692, 691, 525, - /* 640 */ 521, 2704, 2536, 845, 2743, 597, 240, 549, 119, 2706, - /* 650 */ 844, 2708, 2709, 839, 795, 859, 855, 2384, 322, 595, - /* 660 */ 199, 2380, 2797, 2686, 2688, 466, 465, 2793, 2006, 2007, - /* 670 */ 2704, 63, 2111, 2743, 859, 2169, 146, 119, 2706, 844, - /* 680 */ 2708, 2709, 839, 690, 859, 12, 2128, 657, 2531, 2915, - /* 690 */ 183, 2797, 2194, 51, 2845, 465, 2793, 914, 2817, 2067, - /* 700 */ 2068, 2069, 2817, 2817, 2817, 2817, 2817, 1977, 1987, 35, - /* 710 */ 1944, 1789, 1790, 1928, 2180, 40, 39, 2005, 2008, 46, - /* 720 */ 44, 43, 42, 41, 2358, 86, 85, 538, 2438, 2439, - /* 730 */ 229, 2092, 1917, 2444, 1915, 46, 44, 43, 42, 41, - /* 740 */ 98, 435, 2623, 530, 528, 770, 769, 2126, 2127, 2129, - /* 750 */ 2130, 2131, 2442, 12, 289, 10, 413, 439, 288, 517, - /* 760 */ 1788, 1791, 513, 509, 505, 502, 531, 2379, 1920, 1921, - /* 770 */ 1974, 1584, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, - /* 780 */ 1986, 836, 857, 856, 1998, 2000, 2001, 2002, 2003, 2, - /* 790 */ 47, 45, 2009, 1498, 2705, 1497, 754, 480, 469, 236, - /* 800 */ 1916, 500, 180, 865, 2895, 889, 499, 841, 2900, 2852, - /* 810 */ 2664, 487, 2386, 2004, 1585, 1914, 322, 489, 2705, 78, - /* 820 */ 2437, 2439, 2901, 217, 1601, 1944, 2812, 2896, 789, 1499, - /* 830 */ 788, 841, 101, 2865, 636, 2723, 1941, 424, 2895, 1600, - /* 840 */ 451, 2444, 726, 634, 1999, 92, 630, 626, 91, 462, - /* 850 */ 211, 2670, 2807, 840, 829, 754, 787, 217, 1922, 2723, - /* 860 */ 2442, 2896, 789, 2895, 855, 2384, 896, 172, 171, 893, - /* 870 */ 892, 891, 169, 89, 474, 2670, 322, 840, 443, 442, - /* 880 */ 320, 2901, 217, 483, 539, 859, 2896, 789, 959, 2463, - /* 890 */ 2371, 48, 896, 172, 171, 893, 892, 891, 169, 2704, - /* 900 */ 1501, 1502, 2743, 3, 2705, 738, 119, 2706, 844, 2708, - /* 910 */ 2709, 839, 609, 859, 1916, 53, 90, 841, 2915, 2163, - /* 920 */ 2797, 855, 2384, 2704, 465, 2793, 2743, 2006, 2007, 1914, - /* 930 */ 119, 2706, 844, 2708, 2709, 839, 824, 859, 2769, 551, - /* 940 */ 2507, 540, 2915, 796, 2797, 2723, 557, 2507, 465, 2793, - /* 950 */ 472, 864, 863, 862, 855, 2384, 441, 440, 179, 687, - /* 960 */ 2444, 2670, 826, 840, 2769, 2705, 1977, 1987, 473, 2389, - /* 970 */ 855, 2384, 1922, 1478, 661, 1885, 2005, 2008, 841, 2442, - /* 980 */ 765, 689, 2366, 855, 2384, 688, 2444, 740, 2577, 297, - /* 990 */ 559, 1917, 232, 1915, 488, 320, 2120, 1476, 1477, 234, - /* 1000 */ 588, 2507, 959, 573, 1978, 2442, 2723, 479, 478, 2704, - /* 1010 */ 62, 2121, 2743, 1945, 681, 680, 187, 2706, 844, 2708, - /* 1020 */ 2709, 839, 2670, 859, 840, 2091, 2204, 1920, 1921, 1974, - /* 1030 */ 2025, 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, - /* 1040 */ 836, 857, 856, 1998, 2000, 2001, 2002, 2003, 2, 47, - /* 1050 */ 45, 2705, 490, 239, 2119, 581, 580, 469, 382, 1916, - /* 1060 */ 179, 1884, 683, 682, 841, 102, 2888, 1975, 855, 2384, - /* 1070 */ 2704, 2389, 2004, 2743, 1914, 790, 2916, 119, 2706, 844, - /* 1080 */ 2708, 2709, 839, 2670, 859, 702, 701, 2705, 574, 2915, - /* 1090 */ 182, 2797, 2723, 482, 481, 465, 2793, 1605, 2318, 1817, - /* 1100 */ 841, 2203, 2829, 1999, 1650, 1917, 2388, 1915, 2670, 533, - /* 1110 */ 840, 532, 1604, 111, 14, 13, 2202, 1922, 1641, 888, - /* 1120 */ 887, 886, 1645, 885, 1647, 1648, 884, 881, 2723, 1656, - /* 1130 */ 878, 1658, 1659, 875, 872, 869, 2195, 855, 2384, 2377, - /* 1140 */ 2319, 1920, 1921, 531, 2670, 117, 840, 959, 2705, 2899, - /* 1150 */ 48, 2035, 855, 2384, 855, 2384, 2704, 575, 2670, 2743, - /* 1160 */ 2359, 841, 158, 119, 2706, 844, 2708, 2709, 839, 376, - /* 1170 */ 859, 2376, 2381, 2670, 292, 2915, 2201, 2797, 855, 2384, - /* 1180 */ 190, 465, 2793, 212, 148, 611, 2006, 2007, 1480, 2723, - /* 1190 */ 855, 2384, 2704, 322, 1940, 2743, 88, 2200, 300, 119, - /* 1200 */ 2706, 844, 2708, 2709, 839, 2670, 859, 840, 855, 2384, - /* 1210 */ 806, 2915, 2199, 2797, 494, 493, 2198, 465, 2793, 2197, - /* 1220 */ 855, 2384, 855, 2384, 1930, 1977, 1987, 2047, 333, 855, - /* 1230 */ 2384, 2586, 898, 2670, 2858, 2005, 2008, 2004, 34, 1923, - /* 1240 */ 820, 1945, 340, 1941, 1946, 43, 42, 41, 2040, 852, - /* 1250 */ 1917, 2591, 1915, 2704, 2670, 2610, 2743, 855, 2384, 771, - /* 1260 */ 119, 2706, 844, 2708, 2709, 839, 2196, 859, 1999, 2670, - /* 1270 */ 926, 924, 2772, 2670, 2797, 2193, 2670, 853, 465, 2793, - /* 1280 */ 156, 689, 1922, 2768, 2192, 688, 1920, 1921, 1974, 2272, - /* 1290 */ 1976, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 836, - /* 1300 */ 857, 856, 1998, 2000, 2001, 2002, 2003, 2, 47, 45, - /* 1310 */ 2191, 2705, 828, 291, 855, 2384, 469, 2190, 1916, 855, - /* 1320 */ 2384, 2189, 2188, 2670, 841, 308, 179, 2624, 1650, 782, - /* 1330 */ 2187, 2004, 2670, 1914, 368, 890, 792, 2390, 2435, 491, - /* 1340 */ 2186, 2670, 1641, 888, 887, 886, 1645, 885, 1647, 1648, - /* 1350 */ 835, 834, 2723, 1656, 833, 1658, 1659, 832, 872, 869, - /* 1360 */ 894, 2185, 1999, 2435, 706, 705, 704, 2670, 2670, 170, - /* 1370 */ 840, 696, 152, 700, 2670, 316, 1922, 699, 2670, 2670, - /* 1380 */ 2444, 754, 698, 703, 446, 445, 2444, 2670, 697, 2895, - /* 1390 */ 2237, 2444, 444, 693, 692, 691, 2081, 2670, 514, 807, - /* 1400 */ 895, 800, 155, 2435, 2614, 815, 959, 2901, 217, 15, - /* 1410 */ 2443, 383, 2896, 789, 2421, 1931, 2704, 1926, 2670, 2743, - /* 1420 */ 2210, 954, 223, 119, 2706, 844, 2708, 2709, 839, 277, - /* 1430 */ 859, 279, 275, 281, 278, 2770, 280, 2797, 419, 418, - /* 1440 */ 2361, 465, 2793, 950, 283, 2006, 2007, 282, 475, 162, - /* 1450 */ 54, 1934, 1936, 162, 694, 695, 507, 728, 2691, 727, - /* 1460 */ 272, 2004, 2705, 484, 791, 857, 856, 1998, 2000, 2001, - /* 1470 */ 2002, 2003, 1946, 161, 1975, 841, 2256, 191, 1582, 1580, - /* 1480 */ 2254, 2172, 2173, 1562, 1977, 1987, 679, 675, 671, 667, - /* 1490 */ 2245, 271, 1999, 2243, 2005, 2008, 207, 801, 707, 150, - /* 1500 */ 40, 39, 709, 2723, 46, 44, 43, 42, 41, 1917, - /* 1510 */ 761, 1915, 711, 2724, 49, 714, 170, 906, 49, 2670, - /* 1520 */ 200, 840, 2693, 186, 2805, 2806, 1563, 153, 2810, 327, - /* 1530 */ 1833, 2311, 77, 2310, 1841, 64, 2516, 99, 347, 346, - /* 1540 */ 269, 1554, 14, 13, 49, 1920, 1921, 1974, 2226, 1976, - /* 1550 */ 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 836, 857, - /* 1560 */ 856, 1998, 2000, 2001, 2002, 2003, 2, 2704, 49, 736, - /* 1570 */ 2743, 77, 349, 348, 119, 2706, 844, 2708, 2709, 839, - /* 1580 */ 167, 859, 731, 170, 351, 350, 825, 962, 2797, 353, - /* 1590 */ 352, 2115, 465, 2793, 2705, 2125, 74, 805, 867, 2124, - /* 1600 */ 168, 306, 2848, 374, 355, 354, 170, 841, 257, 768, - /* 1610 */ 331, 458, 775, 812, 357, 356, 2041, 268, 454, 952, - /* 1620 */ 206, 259, 266, 793, 149, 1988, 1924, 264, 653, 948, - /* 1630 */ 944, 940, 936, 167, 371, 2723, 754, 809, 498, 1907, - /* 1640 */ 516, 1883, 359, 358, 2895, 907, 256, 361, 360, 1786, - /* 1650 */ 2517, 2670, 1776, 840, 363, 362, 365, 364, 1535, 367, - /* 1660 */ 366, 343, 2901, 217, 1632, 2232, 747, 2896, 789, 1552, - /* 1670 */ 2432, 2705, 783, 477, 476, 1908, 2849, 381, 2859, 1663, - /* 1680 */ 118, 1671, 123, 344, 841, 784, 318, 1678, 713, 857, - /* 1690 */ 856, 1998, 2000, 2001, 2002, 2003, 313, 321, 2345, 2704, - /* 1700 */ 5, 1536, 2743, 725, 2705, 1676, 120, 2706, 844, 2708, - /* 1710 */ 2709, 839, 2723, 859, 173, 816, 501, 841, 506, 287, - /* 1720 */ 2797, 1939, 433, 1949, 2796, 2793, 515, 526, 2670, 527, - /* 1730 */ 840, 224, 529, 225, 2705, 1810, 716, 227, 375, 543, - /* 1740 */ 1940, 550, 238, 710, 708, 2723, 552, 841, 600, 556, - /* 1750 */ 284, 2705, 558, 563, 587, 576, 2509, 589, 613, 612, - /* 1760 */ 342, 2670, 822, 840, 841, 325, 596, 610, 599, 601, - /* 1770 */ 324, 243, 244, 615, 247, 2723, 2704, 617, 618, 2743, - /* 1780 */ 620, 637, 622, 120, 2706, 844, 2708, 2709, 839, 294, - /* 1790 */ 859, 2670, 2723, 840, 1947, 4, 72, 2797, 638, 71, - /* 1800 */ 646, 827, 2793, 645, 1927, 648, 255, 94, 2670, 842, - /* 1810 */ 840, 1942, 2743, 650, 258, 1948, 120, 2706, 844, 2708, - /* 1820 */ 2709, 839, 651, 859, 1950, 652, 739, 261, 2705, 804, - /* 1830 */ 2797, 654, 263, 1951, 428, 2793, 2532, 95, 1952, 2704, - /* 1840 */ 684, 841, 2743, 2526, 96, 2705, 184, 2706, 844, 2708, - /* 1850 */ 2709, 839, 97, 859, 663, 270, 2704, 410, 841, 2743, - /* 1860 */ 717, 718, 124, 120, 2706, 844, 2708, 2709, 839, 2723, - /* 1870 */ 859, 686, 2600, 2374, 274, 2597, 100, 2797, 2370, 276, - /* 1880 */ 754, 175, 2794, 754, 121, 2670, 2723, 840, 2895, 2372, - /* 1890 */ 2367, 2895, 176, 755, 2855, 177, 2596, 730, 377, 163, - /* 1900 */ 732, 293, 2670, 1943, 840, 2578, 2901, 217, 742, 2901, - /* 1910 */ 217, 2896, 789, 741, 2896, 789, 298, 743, 296, 749, - /* 1920 */ 746, 748, 758, 772, 810, 2705, 8, 456, 2864, 2863, - /* 1930 */ 2836, 303, 781, 2704, 307, 189, 2743, 759, 841, 757, - /* 1940 */ 184, 2706, 844, 2708, 2709, 839, 305, 859, 311, 309, - /* 1950 */ 2704, 756, 312, 2743, 2705, 310, 786, 406, 2706, 844, - /* 1960 */ 2708, 2709, 839, 459, 859, 785, 2723, 841, 2918, 794, - /* 1970 */ 797, 2816, 314, 154, 1944, 2894, 317, 2089, 2087, 203, - /* 1980 */ 323, 2813, 2670, 164, 840, 808, 378, 2546, 2856, 2545, - /* 1990 */ 1, 219, 2544, 379, 463, 2723, 813, 814, 818, 165, - /* 2000 */ 61, 336, 821, 2778, 846, 848, 850, 457, 341, 851, - /* 2010 */ 380, 2670, 110, 840, 2385, 2705, 2662, 112, 1455, 2661, - /* 2020 */ 2657, 953, 861, 2656, 2648, 384, 2647, 956, 841, 2639, - /* 2030 */ 2704, 2638, 2654, 2743, 2653, 2645, 370, 406, 2706, 844, - /* 2040 */ 2708, 2709, 839, 2644, 859, 2633, 52, 174, 2632, 2651, - /* 2050 */ 408, 2705, 958, 2650, 734, 2642, 2723, 396, 386, 2704, - /* 2060 */ 2641, 2630, 2743, 2629, 838, 2627, 399, 2706, 844, 2708, - /* 2070 */ 2709, 839, 2670, 859, 840, 420, 2626, 2436, 407, 397, - /* 2080 */ 2622, 388, 2621, 2620, 425, 83, 426, 2615, 503, 504, - /* 2090 */ 1867, 1868, 2723, 222, 2613, 508, 510, 511, 512, 1866, - /* 2100 */ 2612, 2611, 434, 2609, 518, 2608, 520, 2607, 2670, 2606, - /* 2110 */ 840, 522, 524, 1854, 2582, 226, 2581, 780, 228, 1812, - /* 2120 */ 2704, 84, 1813, 2743, 2559, 2558, 2557, 187, 2706, 844, - /* 2130 */ 2708, 2709, 839, 536, 859, 537, 2556, 2555, 2499, 541, - /* 2140 */ 2705, 1749, 2496, 544, 2495, 2489, 2486, 547, 548, 2485, - /* 2150 */ 2484, 231, 2483, 841, 235, 564, 2704, 87, 2488, 2743, - /* 2160 */ 2487, 2482, 233, 405, 2706, 844, 2708, 2709, 839, 2705, - /* 2170 */ 859, 2481, 2763, 2479, 2478, 2477, 2476, 566, 2474, 2473, - /* 2180 */ 2472, 2723, 841, 2471, 2470, 2494, 2469, 2917, 2468, 2467, - /* 2190 */ 2705, 1755, 237, 2456, 2455, 2454, 93, 2670, 2492, 840, - /* 2200 */ 2475, 2466, 2465, 841, 2464, 2462, 2461, 2460, 2459, 2458, - /* 2210 */ 2723, 2457, 2453, 2452, 2524, 2493, 2491, 2451, 2450, 2705, - /* 2220 */ 2449, 2448, 468, 2447, 242, 603, 2670, 2446, 840, 605, - /* 2230 */ 2445, 2723, 841, 2276, 245, 2275, 246, 422, 2274, 2273, - /* 2240 */ 423, 1602, 2271, 248, 1606, 2704, 2268, 2670, 2743, 840, - /* 2250 */ 2267, 470, 406, 2706, 844, 2708, 2709, 839, 249, 859, - /* 2260 */ 2723, 1598, 623, 2260, 2247, 2221, 627, 1479, 625, 631, - /* 2270 */ 2220, 635, 2580, 629, 2704, 262, 2670, 2743, 840, 253, - /* 2280 */ 624, 406, 2706, 844, 2708, 2709, 839, 633, 859, 628, - /* 2290 */ 198, 80, 632, 251, 2576, 729, 2690, 81, 2743, 2566, - /* 2300 */ 209, 2554, 401, 2706, 844, 2708, 2709, 839, 643, 859, - /* 2310 */ 260, 2553, 2530, 2705, 2523, 265, 267, 2362, 1528, 2270, - /* 2320 */ 2266, 664, 666, 665, 2704, 2264, 841, 2743, 668, 669, - /* 2330 */ 2705, 391, 2706, 844, 2708, 2709, 839, 670, 859, 2262, - /* 2340 */ 672, 673, 674, 841, 2259, 676, 678, 2705, 677, 2242, - /* 2350 */ 2240, 2241, 2239, 2217, 2723, 2364, 1682, 2363, 1683, 1588, - /* 2360 */ 841, 1587, 73, 1586, 1583, 1581, 1579, 1578, 1577, 1576, - /* 2370 */ 2670, 2723, 840, 273, 923, 1575, 925, 1572, 1570, 1571, - /* 2380 */ 2257, 447, 1569, 2255, 448, 2246, 449, 2670, 2723, 840, - /* 2390 */ 2244, 450, 2216, 2215, 2214, 715, 2213, 719, 2212, 723, - /* 2400 */ 712, 126, 721, 1848, 2670, 1850, 840, 1847, 2579, 2575, - /* 2410 */ 2705, 1852, 295, 1819, 2565, 28, 2552, 1823, 2704, 67, - /* 2420 */ 2551, 2743, 1838, 841, 733, 390, 2706, 844, 2708, 2709, - /* 2430 */ 839, 2900, 859, 56, 1821, 2704, 20, 744, 2743, 17, - /* 2440 */ 21, 760, 392, 2706, 844, 2708, 2709, 839, 57, 859, - /* 2450 */ 745, 2723, 2704, 2550, 455, 2743, 6, 299, 764, 398, - /* 2460 */ 2706, 844, 2708, 2709, 839, 22, 859, 2670, 1798, 840, - /* 2470 */ 202, 1797, 178, 750, 752, 7, 214, 32, 2691, 2082, - /* 2480 */ 30, 766, 23, 215, 2084, 2142, 302, 65, 2116, 304, - /* 2490 */ 2705, 762, 24, 2123, 2157, 188, 2529, 201, 2156, 31, - /* 2500 */ 460, 2110, 82, 841, 18, 2080, 2161, 2705, 216, 2160, - /* 2510 */ 461, 2162, 59, 319, 2163, 2704, 2064, 193, 2743, 2063, - /* 2520 */ 841, 2528, 402, 2706, 844, 2708, 2709, 839, 105, 859, - /* 2530 */ 106, 2723, 104, 2705, 2522, 326, 107, 2118, 817, 204, - /* 2540 */ 25, 811, 13, 332, 69, 2016, 841, 2670, 2723, 840, - /* 2550 */ 1932, 334, 335, 2015, 58, 11, 2026, 194, 205, 1991, - /* 2560 */ 1990, 874, 877, 1967, 2670, 880, 840, 883, 38, 16, - /* 2570 */ 2705, 2521, 1989, 26, 2723, 843, 1959, 108, 27, 70, - /* 2580 */ 339, 819, 345, 841, 847, 1993, 2178, 109, 2177, 2748, - /* 2590 */ 2670, 2747, 840, 337, 858, 2704, 68, 113, 2743, 2176, - /* 2600 */ 860, 1664, 393, 2706, 844, 2708, 2709, 839, 868, 859, - /* 2610 */ 866, 2723, 2704, 486, 1661, 2743, 870, 1660, 871, 403, - /* 2620 */ 2706, 844, 2708, 2709, 839, 873, 859, 2670, 849, 840, - /* 2630 */ 1657, 2705, 876, 1651, 879, 882, 2175, 1649, 2704, 114, - /* 2640 */ 1655, 2743, 369, 115, 841, 394, 2706, 844, 2708, 2709, - /* 2650 */ 839, 1654, 859, 1677, 2705, 79, 1653, 1673, 1652, 1526, - /* 2660 */ 897, 1566, 1565, 1564, 1561, 1558, 1557, 841, 1556, 1555, - /* 2670 */ 1553, 1551, 2723, 1550, 1549, 2704, 1596, 911, 2743, 1595, - /* 2680 */ 220, 913, 404, 2706, 844, 2708, 2709, 839, 2670, 859, - /* 2690 */ 840, 1547, 1546, 1545, 1544, 2723, 1543, 1542, 1541, 1592, - /* 2700 */ 1590, 1538, 1537, 1534, 1533, 1532, 1531, 2265, 933, 934, - /* 2710 */ 935, 2670, 2263, 840, 937, 2705, 939, 2261, 941, 2258, - /* 2720 */ 943, 945, 938, 2238, 947, 942, 949, 946, 841, 1468, - /* 2730 */ 951, 2211, 1456, 955, 2181, 373, 2704, 2236, 1918, 2743, - /* 2740 */ 2705, 957, 960, 395, 2706, 844, 2708, 2709, 839, 385, - /* 2750 */ 859, 2181, 961, 841, 2181, 2181, 2723, 2181, 2181, 2704, - /* 2760 */ 2181, 2181, 2743, 2181, 2181, 2181, 411, 2706, 844, 2708, - /* 2770 */ 2709, 839, 2670, 859, 840, 2181, 2181, 2181, 2181, 2181, - /* 2780 */ 2181, 2723, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 2790 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2670, 2181, 840, - /* 2800 */ 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 2810 */ 2181, 2181, 2181, 2181, 841, 2181, 2181, 2181, 2181, 2181, - /* 2820 */ 2704, 2181, 2705, 2743, 2181, 2181, 2181, 412, 2706, 844, - /* 2830 */ 2708, 2709, 839, 2181, 859, 841, 2181, 2181, 2181, 2181, - /* 2840 */ 2181, 2181, 2723, 2181, 2181, 2704, 2181, 2181, 2743, 2181, - /* 2850 */ 2181, 2181, 2717, 2706, 844, 2708, 2709, 839, 2670, 859, - /* 2860 */ 840, 2181, 2181, 2723, 2181, 2181, 2181, 2181, 2181, 2181, - /* 2870 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2670, - /* 2880 */ 2181, 840, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181, - /* 2890 */ 2181, 2181, 2181, 2181, 2181, 2181, 841, 2181, 2181, 2181, - /* 2900 */ 2181, 2181, 2181, 2181, 2181, 2181, 2704, 2181, 2181, 2743, - /* 2910 */ 2181, 2181, 2181, 2716, 2706, 844, 2708, 2709, 839, 2181, - /* 2920 */ 859, 2181, 2181, 2181, 2723, 2181, 2181, 2704, 2181, 2181, - /* 2930 */ 2743, 2181, 2181, 2181, 2715, 2706, 844, 2708, 2709, 839, - /* 2940 */ 2670, 859, 840, 2181, 2705, 2181, 2181, 2181, 2181, 2181, - /* 2950 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 841, 2181, 2181, - /* 2960 */ 2181, 2181, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181, - /* 2970 */ 2181, 2181, 2181, 2181, 2181, 2181, 841, 2181, 2181, 2181, - /* 2980 */ 2181, 2181, 2181, 2181, 2181, 2723, 2181, 2181, 2704, 2181, - /* 2990 */ 2181, 2743, 2181, 2181, 2181, 430, 2706, 844, 2708, 2709, - /* 3000 */ 839, 2670, 859, 840, 2723, 2181, 2181, 2181, 2181, 2181, - /* 3010 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3020 */ 2670, 2181, 840, 2181, 2705, 2181, 2181, 2181, 2181, 2181, - /* 3030 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 841, 2181, 2181, - /* 3040 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2704, - /* 3050 */ 2181, 2181, 2743, 2181, 2181, 2181, 431, 2706, 844, 2708, - /* 3060 */ 2709, 839, 2181, 859, 2181, 2723, 2181, 2181, 2704, 2181, - /* 3070 */ 2181, 2743, 2181, 2181, 2181, 427, 2706, 844, 2708, 2709, - /* 3080 */ 839, 2670, 859, 840, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3090 */ 2181, 2181, 2181, 2705, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3100 */ 2181, 2181, 2181, 2181, 2181, 2181, 841, 2181, 2705, 2181, - /* 3110 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3120 */ 2181, 841, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2704, - /* 3130 */ 2181, 2181, 2743, 2181, 2723, 2181, 432, 2706, 844, 2708, - /* 3140 */ 2709, 839, 2181, 859, 2181, 2181, 2181, 2181, 2181, 2723, - /* 3150 */ 2670, 2181, 840, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3160 */ 2181, 2181, 2181, 2181, 2181, 2670, 2181, 840, 2181, 2181, - /* 3170 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3180 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, - /* 3190 */ 2181, 2181, 2181, 2181, 2181, 2181, 2181, 2181, 842, 2181, - /* 3200 */ 2181, 2743, 2181, 2181, 2181, 401, 2706, 844, 2708, 2709, - /* 3210 */ 839, 2181, 859, 2704, 2181, 2181, 2743, 2181, 2181, 2181, - /* 3220 */ 400, 2706, 844, 2708, 2709, 839, 2181, 859, + /* 80 */ 1923, 2270, 802, 155, 2671, 210, 842, 726, 136, 2687, + /* 90 */ 2690, 135, 134, 133, 132, 131, 130, 129, 128, 127, + /* 100 */ 861, 2432, 724, 252, 722, 286, 285, 644, 2705, 2234, + /* 110 */ 961, 2744, 2813, 15, 1943, 119, 2707, 846, 2709, 2710, + /* 120 */ 841, 649, 861, 103, 642, 2226, 197, 199, 856, 2798, + /* 130 */ 646, 2140, 2705, 467, 2794, 2744, 643, 62, 2809, 406, + /* 140 */ 2707, 846, 2709, 2710, 841, 839, 861, 825, 2763, 2007, + /* 150 */ 2008, 50, 439, 218, 2512, 755, 934, 933, 932, 931, + /* 160 */ 499, 2845, 930, 929, 160, 924, 923, 922, 921, 920, + /* 170 */ 919, 918, 159, 912, 911, 910, 498, 497, 907, 906, + /* 180 */ 905, 196, 195, 904, 494, 903, 902, 901, 1978, 1988, + /* 190 */ 804, 185, 2806, 2807, 780, 153, 2811, 290, 2006, 2009, + /* 200 */ 802, 155, 1763, 1764, 192, 2806, 801, 581, 147, 800, + /* 210 */ 790, 2664, 580, 1918, 487, 1916, 2896, 9, 2896, 2100, + /* 220 */ 579, 1927, 40, 39, 739, 1942, 46, 44, 43, 42, + /* 230 */ 41, 2361, 2706, 2206, 789, 217, 789, 217, 122, 2897, + /* 240 */ 791, 2897, 791, 1690, 1691, 843, 2389, 489, 2387, 1921, + /* 250 */ 1922, 1975, 779, 1977, 1980, 1981, 1982, 1983, 1984, 1985, + /* 260 */ 1986, 1987, 838, 859, 858, 1999, 2001, 2002, 2003, 2004, + /* 270 */ 2, 47, 45, 2724, 917, 476, 415, 2345, 1940, 471, + /* 280 */ 2724, 1917, 1844, 1845, 2901, 588, 861, 431, 609, 2671, + /* 290 */ 2671, 842, 2896, 608, 2005, 60, 1915, 40, 39, 455, + /* 300 */ 2586, 46, 44, 43, 42, 41, 753, 2015, 736, 1946, + /* 310 */ 567, 2900, 610, 1942, 197, 2897, 2899, 416, 569, 1499, + /* 320 */ 322, 1498, 125, 2806, 2807, 2000, 153, 2811, 180, 547, + /* 330 */ 857, 2385, 19, 664, 322, 2034, 151, 2705, 2388, 1923, + /* 340 */ 2744, 2071, 2513, 775, 119, 2707, 846, 2709, 2710, 841, + /* 350 */ 221, 861, 1472, 778, 157, 1500, 166, 2769, 2798, 2183, + /* 360 */ 40, 39, 467, 2794, 46, 44, 43, 42, 41, 961, + /* 370 */ 440, 1479, 15, 898, 172, 171, 895, 894, 893, 169, + /* 380 */ 1946, 555, 1942, 145, 144, 143, 142, 141, 140, 139, + /* 390 */ 138, 137, 657, 1979, 1474, 1477, 1478, 537, 62, 40, + /* 400 */ 39, 2035, 536, 46, 44, 43, 42, 41, 2007, 2008, + /* 410 */ 661, 2519, 2498, 660, 596, 595, 594, 593, 592, 587, + /* 420 */ 586, 585, 584, 423, 29, 1975, 2071, 574, 573, 572, + /* 430 */ 571, 570, 564, 563, 562, 254, 557, 556, 438, 644, + /* 440 */ 856, 2234, 548, 1751, 1752, 329, 330, 1978, 1988, 1770, + /* 450 */ 328, 756, 781, 776, 769, 765, 1976, 2006, 2009, 2896, + /* 460 */ 2818, 2068, 2069, 2070, 2818, 2818, 2818, 2818, 2818, 2526, + /* 470 */ 2376, 662, 1918, 521, 1916, 62, 900, 2902, 217, 213, + /* 480 */ 740, 1979, 2897, 791, 857, 2385, 857, 2385, 36, 469, + /* 490 */ 2029, 2030, 2031, 2032, 2033, 2037, 2038, 2039, 2040, 2073, + /* 500 */ 2074, 2075, 2076, 2077, 146, 2141, 55, 377, 1921, 1922, + /* 510 */ 1975, 687, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, + /* 520 */ 1987, 838, 859, 858, 1999, 2001, 2002, 2003, 2004, 2, + /* 530 */ 12, 47, 45, 50, 476, 1926, 2706, 2539, 181, 471, + /* 540 */ 1947, 1917, 116, 390, 1976, 861, 2068, 2069, 2070, 805, + /* 550 */ 208, 113, 473, 600, 2005, 1590, 1915, 2171, 2536, 847, + /* 560 */ 388, 76, 651, 2578, 75, 2706, 40, 39, 454, 2586, + /* 570 */ 46, 44, 43, 42, 41, 417, 737, 2724, 843, 2071, + /* 580 */ 2236, 322, 1943, 1923, 2681, 2000, 2901, 250, 623, 621, + /* 590 */ 618, 616, 19, 2671, 2896, 842, 40, 39, 1592, 1923, + /* 600 */ 46, 44, 43, 42, 41, 2491, 2724, 708, 707, 706, + /* 610 */ 525, 1947, 2685, 2900, 698, 152, 702, 2897, 2898, 241, + /* 620 */ 701, 2539, 2671, 658, 842, 700, 705, 448, 447, 961, + /* 630 */ 914, 699, 15, 62, 98, 446, 695, 694, 693, 527, + /* 640 */ 523, 2705, 2537, 847, 2744, 599, 240, 551, 119, 2707, + /* 650 */ 846, 2709, 2710, 841, 797, 861, 857, 2385, 322, 597, + /* 660 */ 199, 2381, 2798, 2687, 2689, 468, 467, 2794, 2007, 2008, + /* 670 */ 2705, 63, 2112, 2744, 861, 2170, 146, 119, 2707, 846, + /* 680 */ 2709, 2710, 841, 692, 861, 12, 2129, 659, 2532, 2916, + /* 690 */ 183, 2798, 2195, 51, 2846, 467, 2794, 916, 2818, 2068, + /* 700 */ 2069, 2070, 2818, 2818, 2818, 2818, 2818, 1978, 1988, 35, + /* 710 */ 1945, 1790, 1791, 1929, 2181, 40, 39, 2006, 2009, 46, + /* 720 */ 44, 43, 42, 41, 2359, 86, 85, 540, 2439, 2440, + /* 730 */ 229, 2093, 1918, 2445, 1916, 46, 44, 43, 42, 41, + /* 740 */ 98, 437, 2624, 532, 530, 772, 771, 2127, 2128, 2130, + /* 750 */ 2131, 2132, 2443, 12, 289, 10, 414, 441, 288, 519, + /* 760 */ 1789, 1792, 515, 511, 507, 504, 533, 2380, 1921, 1922, + /* 770 */ 1975, 1585, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, + /* 780 */ 1987, 838, 859, 858, 1999, 2001, 2002, 2003, 2004, 2, + /* 790 */ 47, 45, 2010, 1499, 2706, 1498, 756, 482, 471, 236, + /* 800 */ 1917, 502, 180, 867, 2896, 891, 501, 843, 2901, 2853, + /* 810 */ 2665, 489, 2387, 2005, 1586, 1915, 322, 491, 2706, 78, + /* 820 */ 2438, 2440, 2902, 217, 1602, 1945, 2813, 2897, 791, 1500, + /* 830 */ 790, 843, 101, 2866, 638, 2724, 1942, 426, 2896, 1601, + /* 840 */ 453, 2445, 728, 636, 2000, 92, 632, 628, 91, 464, + /* 850 */ 211, 2671, 2808, 842, 831, 756, 789, 217, 1923, 2724, + /* 860 */ 2443, 2897, 791, 2896, 857, 2385, 898, 172, 171, 895, + /* 870 */ 894, 893, 169, 89, 476, 2671, 322, 842, 445, 444, + /* 880 */ 320, 2902, 217, 485, 541, 861, 2897, 791, 961, 2464, + /* 890 */ 2372, 48, 898, 172, 171, 895, 894, 893, 169, 2705, + /* 900 */ 1502, 1503, 2744, 3, 2706, 740, 119, 2707, 846, 2709, + /* 910 */ 2710, 841, 611, 861, 1917, 53, 90, 843, 2916, 2164, + /* 920 */ 2798, 857, 2385, 2705, 467, 2794, 2744, 2007, 2008, 1915, + /* 930 */ 119, 2707, 846, 2709, 2710, 841, 826, 861, 2770, 553, + /* 940 */ 2508, 542, 2916, 798, 2798, 2724, 559, 2508, 467, 2794, + /* 950 */ 474, 866, 865, 864, 857, 2385, 443, 442, 179, 689, + /* 960 */ 2445, 2671, 828, 842, 2770, 2706, 1978, 1988, 475, 2390, + /* 970 */ 857, 2385, 1923, 1479, 663, 1886, 2006, 2009, 843, 2443, + /* 980 */ 767, 691, 376, 857, 2385, 690, 2445, 742, 2578, 297, + /* 990 */ 561, 1918, 232, 1916, 490, 320, 2121, 1477, 1478, 234, + /* 1000 */ 590, 2508, 961, 575, 1979, 2443, 2724, 481, 480, 2705, + /* 1010 */ 62, 2122, 2744, 1946, 683, 682, 187, 2707, 846, 2709, + /* 1020 */ 2710, 841, 2671, 861, 842, 2092, 2205, 1921, 1922, 1975, + /* 1030 */ 2026, 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, + /* 1040 */ 838, 859, 858, 1999, 2001, 2002, 2003, 2004, 2, 47, + /* 1050 */ 45, 2706, 492, 239, 2120, 583, 582, 471, 383, 1917, + /* 1060 */ 179, 1885, 685, 684, 843, 102, 2889, 1976, 857, 2385, + /* 1070 */ 2705, 2390, 2005, 2744, 1915, 792, 2917, 119, 2707, 846, + /* 1080 */ 2709, 2710, 841, 2671, 861, 704, 703, 2706, 576, 2916, + /* 1090 */ 182, 2798, 2724, 484, 483, 467, 2794, 1606, 2319, 1818, + /* 1100 */ 843, 2204, 2830, 2000, 1651, 1918, 2389, 1916, 2671, 535, + /* 1110 */ 842, 534, 1605, 111, 14, 13, 2203, 1923, 1642, 890, + /* 1120 */ 889, 888, 1646, 887, 1648, 1649, 886, 883, 2724, 1657, + /* 1130 */ 880, 1659, 1660, 877, 874, 871, 2196, 857, 2385, 2378, + /* 1140 */ 2320, 1921, 1922, 533, 2671, 117, 842, 961, 2706, 2900, + /* 1150 */ 48, 2036, 857, 2385, 857, 2385, 2705, 577, 2671, 2744, + /* 1160 */ 2360, 843, 158, 119, 2707, 846, 2709, 2710, 841, 377, + /* 1170 */ 861, 2377, 2382, 2671, 292, 2916, 2202, 2798, 857, 2385, + /* 1180 */ 190, 467, 2794, 212, 148, 613, 2007, 2008, 1481, 2724, + /* 1190 */ 857, 2385, 2705, 322, 1941, 2744, 88, 2201, 300, 119, + /* 1200 */ 2707, 846, 2709, 2710, 841, 2671, 861, 842, 857, 2385, + /* 1210 */ 808, 2916, 2200, 2798, 496, 495, 2199, 467, 2794, 2198, + /* 1220 */ 857, 2385, 857, 2385, 1931, 1978, 1988, 2048, 333, 857, + /* 1230 */ 2385, 2587, 900, 2671, 2859, 2006, 2009, 2005, 34, 1924, + /* 1240 */ 822, 1946, 340, 1942, 1947, 43, 42, 41, 2041, 854, + /* 1250 */ 1918, 2592, 1916, 2705, 2671, 2611, 2744, 857, 2385, 773, + /* 1260 */ 119, 2707, 846, 2709, 2710, 841, 2197, 861, 2000, 2671, + /* 1270 */ 928, 926, 2773, 2671, 2798, 2194, 2671, 855, 467, 2794, + /* 1280 */ 156, 691, 1923, 2769, 2193, 690, 1921, 1922, 1975, 2273, + /* 1290 */ 1977, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 838, + /* 1300 */ 859, 858, 1999, 2001, 2002, 2003, 2004, 2, 47, 45, + /* 1310 */ 2192, 2706, 830, 291, 857, 2385, 471, 2191, 1917, 857, + /* 1320 */ 2385, 2190, 2189, 2671, 843, 308, 179, 2625, 1651, 784, + /* 1330 */ 2188, 2005, 2671, 1915, 368, 892, 794, 2391, 2436, 493, + /* 1340 */ 2187, 2671, 1642, 890, 889, 888, 1646, 887, 1648, 1649, + /* 1350 */ 837, 836, 2724, 1657, 835, 1659, 1660, 834, 874, 871, + /* 1360 */ 896, 2186, 2000, 2436, 708, 707, 706, 2671, 2671, 170, + /* 1370 */ 842, 698, 152, 702, 2671, 316, 1923, 701, 2671, 2671, + /* 1380 */ 2445, 756, 700, 705, 448, 447, 2445, 2671, 699, 2896, + /* 1390 */ 2238, 2445, 446, 695, 694, 693, 2082, 2671, 516, 809, + /* 1400 */ 897, 802, 155, 2436, 2615, 817, 961, 2902, 217, 15, + /* 1410 */ 2444, 384, 2897, 791, 2422, 1932, 2705, 1927, 2671, 2744, + /* 1420 */ 2211, 956, 223, 119, 2707, 846, 2709, 2710, 841, 277, + /* 1430 */ 861, 279, 275, 281, 278, 2771, 280, 2798, 421, 420, + /* 1440 */ 2362, 467, 2794, 952, 283, 2007, 2008, 282, 477, 162, + /* 1450 */ 54, 1935, 1937, 162, 696, 697, 509, 730, 2692, 729, + /* 1460 */ 272, 2005, 2706, 486, 793, 859, 858, 1999, 2001, 2002, + /* 1470 */ 2003, 2004, 1947, 161, 1976, 843, 2257, 191, 1583, 1581, + /* 1480 */ 2255, 2173, 2174, 1563, 1978, 1988, 681, 677, 673, 669, + /* 1490 */ 2246, 271, 2000, 2244, 2006, 2009, 207, 803, 709, 150, + /* 1500 */ 40, 39, 711, 2724, 46, 44, 43, 42, 41, 1918, + /* 1510 */ 763, 1916, 713, 2725, 49, 716, 170, 908, 49, 2671, + /* 1520 */ 200, 842, 2694, 186, 2806, 2807, 1564, 153, 2811, 327, + /* 1530 */ 1834, 2312, 77, 2311, 1842, 64, 2517, 99, 347, 346, + /* 1540 */ 269, 1555, 14, 13, 49, 1921, 1922, 1975, 2227, 1977, + /* 1550 */ 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 838, 859, + /* 1560 */ 858, 1999, 2001, 2002, 2003, 2004, 2, 2705, 49, 738, + /* 1570 */ 2744, 77, 349, 348, 119, 2707, 846, 2709, 2710, 841, + /* 1580 */ 167, 861, 733, 170, 351, 350, 827, 964, 2798, 353, + /* 1590 */ 352, 2116, 467, 2794, 2706, 2126, 74, 807, 869, 2125, + /* 1600 */ 168, 306, 2849, 374, 355, 354, 170, 843, 257, 770, + /* 1610 */ 331, 460, 777, 814, 357, 356, 2042, 268, 456, 954, + /* 1620 */ 206, 259, 266, 795, 149, 1989, 1925, 264, 655, 950, + /* 1630 */ 946, 942, 938, 167, 371, 2724, 756, 811, 500, 1908, + /* 1640 */ 518, 1884, 359, 358, 2896, 909, 256, 361, 360, 1787, + /* 1650 */ 2518, 2671, 1777, 842, 363, 362, 365, 364, 1536, 367, + /* 1660 */ 366, 343, 2902, 217, 1633, 2233, 749, 2897, 791, 1553, + /* 1670 */ 2433, 2706, 785, 479, 478, 1909, 2850, 382, 2860, 1664, + /* 1680 */ 118, 1672, 123, 344, 843, 786, 318, 1679, 715, 859, + /* 1690 */ 858, 1999, 2001, 2002, 2003, 2004, 313, 321, 2346, 2705, + /* 1700 */ 5, 1537, 2744, 727, 2706, 1677, 120, 2707, 846, 2709, + /* 1710 */ 2710, 841, 2724, 861, 173, 818, 503, 843, 508, 287, + /* 1720 */ 2798, 1940, 435, 1950, 2797, 2794, 517, 528, 2671, 529, + /* 1730 */ 842, 224, 531, 225, 2706, 1811, 718, 227, 375, 545, + /* 1740 */ 1941, 552, 238, 712, 710, 2724, 554, 843, 602, 558, + /* 1750 */ 284, 2706, 560, 565, 589, 578, 2510, 591, 615, 614, + /* 1760 */ 342, 2671, 824, 842, 843, 325, 598, 612, 601, 603, + /* 1770 */ 324, 243, 244, 617, 247, 2724, 2705, 619, 620, 2744, + /* 1780 */ 622, 639, 624, 120, 2707, 846, 2709, 2710, 841, 294, + /* 1790 */ 861, 2671, 2724, 842, 1948, 4, 72, 2798, 640, 71, + /* 1800 */ 648, 829, 2794, 647, 1928, 650, 255, 94, 2671, 844, + /* 1810 */ 842, 1943, 2744, 652, 258, 1949, 120, 2707, 846, 2709, + /* 1820 */ 2710, 841, 653, 861, 1951, 654, 741, 261, 2706, 806, + /* 1830 */ 2798, 656, 263, 1952, 430, 2794, 2533, 95, 1953, 2705, + /* 1840 */ 686, 843, 2744, 2527, 96, 2706, 184, 2707, 846, 2709, + /* 1850 */ 2710, 841, 97, 861, 665, 270, 2705, 411, 843, 2744, + /* 1860 */ 719, 720, 124, 120, 2707, 846, 2709, 2710, 841, 2724, + /* 1870 */ 861, 688, 2601, 2375, 274, 2598, 100, 2798, 2371, 276, + /* 1880 */ 756, 175, 2795, 756, 121, 2671, 2724, 842, 2896, 2373, + /* 1890 */ 2368, 2896, 176, 757, 2856, 177, 2597, 732, 378, 163, + /* 1900 */ 734, 293, 2671, 1944, 842, 2579, 2902, 217, 744, 2902, + /* 1910 */ 217, 2897, 791, 743, 2897, 791, 298, 745, 296, 751, + /* 1920 */ 748, 750, 760, 774, 812, 2706, 8, 458, 2865, 2864, + /* 1930 */ 2837, 303, 783, 2705, 307, 189, 2744, 761, 843, 759, + /* 1940 */ 184, 2707, 846, 2709, 2710, 841, 305, 861, 311, 309, + /* 1950 */ 2705, 758, 312, 2744, 2706, 310, 788, 407, 2707, 846, + /* 1960 */ 2709, 2710, 841, 461, 861, 787, 2724, 843, 2919, 796, + /* 1970 */ 799, 2817, 314, 154, 1945, 2895, 317, 2090, 2088, 203, + /* 1980 */ 323, 2814, 2671, 164, 842, 810, 379, 2547, 2857, 2546, + /* 1990 */ 1, 219, 2545, 380, 465, 2724, 815, 816, 820, 165, + /* 2000 */ 61, 336, 823, 2779, 848, 850, 852, 459, 341, 853, + /* 2010 */ 381, 2671, 110, 842, 2386, 2706, 2663, 112, 1456, 2662, + /* 2020 */ 2658, 955, 863, 2657, 2649, 385, 2648, 958, 843, 2640, + /* 2030 */ 2705, 2639, 2655, 2744, 2654, 2646, 370, 407, 2707, 846, + /* 2040 */ 2709, 2710, 841, 2645, 861, 2634, 52, 174, 2633, 2652, + /* 2050 */ 419, 2706, 960, 2651, 736, 2643, 2724, 397, 387, 2705, + /* 2060 */ 2642, 2631, 2744, 2630, 840, 2628, 400, 2707, 846, 2709, + /* 2070 */ 2710, 841, 2671, 861, 842, 422, 2627, 2437, 408, 398, + /* 2080 */ 409, 389, 2623, 2622, 427, 2621, 83, 2616, 505, 428, + /* 2090 */ 506, 1868, 2724, 1869, 222, 510, 2614, 512, 513, 514, + /* 2100 */ 1867, 2613, 2612, 436, 2610, 520, 2609, 522, 2671, 2608, + /* 2110 */ 842, 524, 2607, 526, 1855, 2583, 226, 782, 2582, 228, + /* 2120 */ 2705, 84, 1814, 2744, 1813, 2560, 2559, 187, 2707, 846, + /* 2130 */ 2709, 2710, 841, 2558, 861, 538, 539, 2557, 2556, 2500, + /* 2140 */ 2706, 543, 1750, 2497, 546, 2496, 2490, 549, 2487, 550, + /* 2150 */ 2486, 2485, 231, 843, 233, 2488, 2705, 2484, 87, 2744, + /* 2160 */ 2489, 2483, 2482, 406, 2707, 846, 2709, 2710, 841, 2706, + /* 2170 */ 861, 2480, 2764, 2479, 2478, 235, 566, 2477, 568, 2475, + /* 2180 */ 2474, 2724, 843, 2473, 2472, 2471, 2495, 2918, 2470, 2469, + /* 2190 */ 2706, 237, 2457, 2456, 93, 2455, 2454, 2671, 2468, 842, + /* 2200 */ 2493, 2476, 2467, 843, 2466, 2465, 2463, 2462, 2461, 2460, + /* 2210 */ 2724, 2459, 2458, 2453, 2525, 2494, 2492, 2452, 2451, 2706, + /* 2220 */ 2450, 1756, 470, 242, 2449, 605, 2671, 2448, 842, 607, + /* 2230 */ 2447, 2724, 843, 2446, 2277, 245, 2276, 424, 246, 2275, + /* 2240 */ 425, 1603, 248, 1607, 2274, 2705, 2272, 2671, 2744, 842, + /* 2250 */ 2269, 472, 407, 2707, 846, 2709, 2710, 841, 2268, 861, + /* 2260 */ 2724, 2261, 1599, 249, 627, 625, 629, 2248, 631, 633, + /* 2270 */ 2222, 1480, 2221, 635, 2705, 626, 2671, 2744, 842, 637, + /* 2280 */ 80, 407, 2707, 846, 2709, 2710, 841, 630, 861, 634, + /* 2290 */ 251, 81, 198, 2581, 2577, 731, 2691, 209, 2744, 253, + /* 2300 */ 645, 2567, 402, 2707, 846, 2709, 2710, 841, 2555, 861, + /* 2310 */ 260, 2554, 262, 2706, 265, 2531, 2524, 2363, 267, 2271, + /* 2320 */ 2267, 666, 668, 667, 2705, 2265, 843, 2744, 1529, 670, + /* 2330 */ 2706, 392, 2707, 846, 2709, 2710, 841, 671, 861, 672, + /* 2340 */ 2263, 674, 675, 843, 676, 2260, 678, 2706, 679, 680, + /* 2350 */ 2243, 2241, 2242, 2240, 2724, 2218, 2365, 1683, 1684, 2364, + /* 2360 */ 843, 73, 273, 1589, 1588, 1587, 1584, 1582, 1580, 1579, + /* 2370 */ 2671, 2724, 842, 2258, 1578, 1577, 1576, 925, 2256, 927, + /* 2380 */ 1573, 2247, 1571, 2245, 1572, 1570, 449, 2671, 2724, 842, + /* 2390 */ 450, 451, 452, 2217, 717, 2216, 714, 2215, 2214, 721, + /* 2400 */ 723, 2213, 725, 1849, 2671, 126, 842, 2580, 1851, 1848, + /* 2410 */ 2706, 2576, 1853, 28, 295, 2566, 67, 2553, 2705, 1820, + /* 2420 */ 1839, 2744, 1822, 843, 746, 391, 2707, 846, 2709, 2710, + /* 2430 */ 841, 56, 861, 2552, 2901, 2705, 20, 17, 2744, 30, + /* 2440 */ 6, 762, 393, 2707, 846, 2709, 2710, 841, 2143, 861, + /* 2450 */ 57, 2724, 2705, 457, 747, 2744, 1799, 21, 299, 399, + /* 2460 */ 2707, 846, 2709, 2710, 841, 766, 861, 2671, 1798, 842, + /* 2470 */ 7, 302, 22, 752, 754, 1824, 214, 202, 735, 178, + /* 2480 */ 32, 2117, 23, 2692, 768, 2083, 215, 764, 2085, 188, + /* 2490 */ 2706, 304, 65, 2124, 24, 2158, 2157, 201, 319, 31, + /* 2500 */ 462, 2111, 82, 843, 18, 2081, 2163, 2706, 2164, 216, + /* 2510 */ 2162, 2161, 463, 2065, 59, 2705, 193, 2551, 2744, 2530, + /* 2520 */ 843, 104, 403, 2707, 846, 2709, 2710, 841, 2064, 861, + /* 2530 */ 105, 2724, 326, 2706, 2529, 2119, 204, 106, 332, 69, + /* 2540 */ 2523, 335, 813, 819, 107, 25, 843, 2671, 2724, 842, + /* 2550 */ 13, 2017, 2016, 11, 1933, 2027, 1992, 194, 205, 1991, + /* 2560 */ 1990, 58, 1960, 876, 2671, 879, 842, 882, 885, 38, + /* 2570 */ 2706, 1968, 2522, 16, 2724, 26, 845, 27, 70, 108, + /* 2580 */ 849, 339, 109, 843, 345, 334, 1994, 868, 2179, 2749, + /* 2590 */ 2671, 2748, 842, 113, 1665, 2705, 860, 68, 2744, 821, + /* 2600 */ 862, 337, 394, 2707, 846, 2709, 2710, 841, 2178, 861, + /* 2610 */ 488, 2724, 2705, 2177, 870, 2744, 872, 1662, 873, 404, + /* 2620 */ 2707, 846, 2709, 2710, 841, 851, 861, 2671, 1661, 842, + /* 2630 */ 875, 2706, 1658, 878, 1652, 881, 2176, 1650, 2705, 884, + /* 2640 */ 1656, 2744, 114, 369, 843, 395, 2707, 846, 2709, 2710, + /* 2650 */ 841, 115, 861, 1678, 2706, 1655, 1654, 79, 1653, 1674, + /* 2660 */ 1567, 1527, 899, 1566, 1565, 1562, 1559, 843, 1558, 1557, + /* 2670 */ 1556, 1554, 2724, 1552, 1597, 2705, 1551, 1550, 2744, 1596, + /* 2680 */ 220, 913, 405, 2707, 846, 2709, 2710, 841, 2671, 861, + /* 2690 */ 842, 1548, 915, 1547, 1546, 2724, 1545, 1544, 1543, 1542, + /* 2700 */ 1593, 1591, 1539, 1538, 1535, 1534, 1533, 1532, 2266, 935, + /* 2710 */ 936, 2671, 2264, 842, 939, 2706, 2262, 937, 941, 943, + /* 2720 */ 945, 940, 2259, 944, 947, 948, 2239, 949, 843, 951, + /* 2730 */ 2237, 953, 1469, 2212, 957, 1457, 2705, 373, 959, 2744, + /* 2740 */ 2706, 1919, 386, 396, 2707, 846, 2709, 2710, 841, 962, + /* 2750 */ 861, 963, 2182, 843, 2182, 2182, 2724, 2182, 2182, 2705, + /* 2760 */ 2182, 2182, 2744, 2182, 2182, 2182, 412, 2707, 846, 2709, + /* 2770 */ 2710, 841, 2671, 861, 842, 2182, 2182, 2182, 2182, 2182, + /* 2780 */ 2182, 2724, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 2790 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2671, 2182, 842, + /* 2800 */ 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 2810 */ 2182, 2182, 2182, 2182, 843, 2182, 2182, 2182, 2182, 2182, + /* 2820 */ 2705, 2182, 2706, 2744, 2182, 2182, 2182, 413, 2707, 846, + /* 2830 */ 2709, 2710, 841, 2182, 861, 843, 2182, 2182, 2182, 2182, + /* 2840 */ 2182, 2182, 2724, 2182, 2182, 2705, 2182, 2182, 2744, 2182, + /* 2850 */ 2182, 2182, 2718, 2707, 846, 2709, 2710, 841, 2671, 861, + /* 2860 */ 842, 2182, 2182, 2724, 2182, 2182, 2182, 2182, 2182, 2182, + /* 2870 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2671, + /* 2880 */ 2182, 842, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182, + /* 2890 */ 2182, 2182, 2182, 2182, 2182, 2182, 843, 2182, 2182, 2182, + /* 2900 */ 2182, 2182, 2182, 2182, 2182, 2182, 2705, 2182, 2182, 2744, + /* 2910 */ 2182, 2182, 2182, 2717, 2707, 846, 2709, 2710, 841, 2182, + /* 2920 */ 861, 2182, 2182, 2182, 2724, 2182, 2182, 2705, 2182, 2182, + /* 2930 */ 2744, 2182, 2182, 2182, 2716, 2707, 846, 2709, 2710, 841, + /* 2940 */ 2671, 861, 842, 2182, 2706, 2182, 2182, 2182, 2182, 2182, + /* 2950 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 843, 2182, 2182, + /* 2960 */ 2182, 2182, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182, + /* 2970 */ 2182, 2182, 2182, 2182, 2182, 2182, 843, 2182, 2182, 2182, + /* 2980 */ 2182, 2182, 2182, 2182, 2182, 2724, 2182, 2182, 2705, 2182, + /* 2990 */ 2182, 2744, 2182, 2182, 2182, 432, 2707, 846, 2709, 2710, + /* 3000 */ 841, 2671, 861, 842, 2724, 2182, 2182, 2182, 2182, 2182, + /* 3010 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3020 */ 2671, 2182, 842, 2182, 2706, 2182, 2182, 2182, 2182, 2182, + /* 3030 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 843, 2182, 2182, + /* 3040 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2705, + /* 3050 */ 2182, 2182, 2744, 2182, 2182, 2182, 433, 2707, 846, 2709, + /* 3060 */ 2710, 841, 2182, 861, 2182, 2724, 2182, 2182, 2705, 2182, + /* 3070 */ 2182, 2744, 2182, 2182, 2182, 429, 2707, 846, 2709, 2710, + /* 3080 */ 841, 2671, 861, 842, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3090 */ 2182, 2182, 2182, 2706, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3100 */ 2182, 2182, 2182, 2182, 2182, 2182, 843, 2182, 2706, 2182, + /* 3110 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3120 */ 2182, 843, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2705, + /* 3130 */ 2182, 2182, 2744, 2182, 2724, 2182, 434, 2707, 846, 2709, + /* 3140 */ 2710, 841, 2182, 861, 2182, 2182, 2182, 2182, 2182, 2724, + /* 3150 */ 2671, 2182, 842, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3160 */ 2182, 2182, 2182, 2182, 2182, 2671, 2182, 842, 2182, 2182, + /* 3170 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3180 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, + /* 3190 */ 2182, 2182, 2182, 2182, 2182, 2182, 2182, 2182, 844, 2182, + /* 3200 */ 2182, 2744, 2182, 2182, 2182, 402, 2707, 846, 2709, 2710, + /* 3210 */ 841, 2182, 861, 2705, 2182, 2182, 2744, 2182, 2182, 2182, + /* 3220 */ 401, 2707, 846, 2709, 2710, 841, 2182, 861, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 506, 507, 438, 379, 386, 391, 421, 389, 390, 519, @@ -1127,77 +1130,77 @@ static const YYCODETYPE yy_lookahead[] = { /* 2020 */ 436, 38, 425, 436, 436, 391, 436, 381, 392, 436, /* 2030 */ 484, 436, 436, 487, 436, 436, 398, 491, 492, 493, /* 2040 */ 494, 495, 496, 436, 498, 436, 467, 385, 436, 436, - /* 2050 */ 476, 379, 384, 436, 483, 436, 420, 414, 399, 484, + /* 2050 */ 422, 379, 384, 436, 483, 436, 420, 414, 399, 484, /* 2060 */ 436, 436, 487, 436, 392, 436, 491, 492, 493, 494, /* 2070 */ 495, 496, 436, 498, 438, 422, 436, 436, 414, 414, - /* 2080 */ 0, 377, 0, 0, 458, 45, 458, 0, 37, 240, - /* 2090 */ 37, 37, 420, 37, 0, 240, 37, 37, 240, 37, - /* 2100 */ 0, 0, 240, 0, 37, 0, 37, 0, 436, 0, - /* 2110 */ 438, 22, 37, 235, 0, 221, 0, 542, 221, 213, - /* 2120 */ 484, 222, 215, 487, 0, 0, 0, 491, 492, 493, - /* 2130 */ 494, 495, 496, 209, 498, 208, 0, 0, 154, 49, - /* 2140 */ 379, 49, 0, 37, 0, 0, 0, 37, 52, 0, - /* 2150 */ 0, 49, 0, 392, 173, 37, 484, 45, 0, 487, - /* 2160 */ 0, 0, 49, 491, 492, 493, 494, 495, 496, 379, - /* 2170 */ 498, 0, 500, 0, 0, 0, 0, 173, 0, 0, + /* 2080 */ 476, 377, 0, 0, 458, 0, 45, 0, 37, 458, + /* 2090 */ 240, 37, 420, 37, 37, 240, 0, 37, 37, 240, + /* 2100 */ 37, 0, 0, 240, 0, 37, 0, 37, 436, 0, + /* 2110 */ 438, 22, 0, 37, 235, 0, 221, 542, 0, 221, + /* 2120 */ 484, 222, 215, 487, 213, 0, 0, 491, 492, 493, + /* 2130 */ 494, 495, 496, 0, 498, 209, 208, 0, 0, 154, + /* 2140 */ 379, 49, 49, 0, 37, 0, 0, 37, 0, 52, + /* 2150 */ 0, 0, 49, 392, 49, 0, 484, 0, 45, 487, + /* 2160 */ 0, 0, 0, 491, 492, 493, 494, 495, 496, 379, + /* 2170 */ 498, 0, 500, 0, 0, 173, 37, 0, 173, 0, /* 2180 */ 0, 420, 392, 0, 0, 0, 0, 551, 0, 0, - /* 2190 */ 379, 22, 49, 0, 0, 0, 45, 436, 0, 438, + /* 2190 */ 379, 49, 0, 0, 45, 0, 0, 436, 0, 438, /* 2200 */ 0, 0, 0, 392, 0, 0, 0, 0, 0, 0, /* 2210 */ 420, 0, 0, 0, 0, 0, 0, 0, 0, 379, - /* 2220 */ 0, 0, 461, 0, 154, 153, 436, 0, 438, 152, - /* 2230 */ 0, 420, 392, 0, 66, 0, 66, 50, 0, 0, - /* 2240 */ 50, 22, 0, 66, 22, 484, 0, 436, 487, 438, - /* 2250 */ 0, 461, 491, 492, 493, 494, 495, 496, 66, 498, - /* 2260 */ 420, 37, 37, 0, 0, 0, 37, 14, 42, 37, - /* 2270 */ 0, 37, 0, 42, 484, 204, 436, 487, 438, 43, - /* 2280 */ 52, 491, 492, 493, 494, 495, 496, 42, 498, 52, - /* 2290 */ 33, 42, 52, 45, 0, 484, 49, 42, 487, 0, - /* 2300 */ 49, 0, 491, 492, 493, 494, 495, 496, 49, 498, - /* 2310 */ 42, 0, 0, 379, 0, 49, 49, 0, 74, 0, - /* 2320 */ 0, 37, 42, 52, 484, 0, 392, 487, 37, 52, - /* 2330 */ 379, 491, 492, 493, 494, 495, 496, 42, 498, 0, - /* 2340 */ 37, 52, 42, 392, 0, 37, 42, 379, 52, 0, - /* 2350 */ 0, 0, 0, 0, 420, 0, 22, 0, 37, 22, - /* 2360 */ 392, 37, 121, 37, 37, 37, 37, 37, 37, 37, - /* 2370 */ 436, 420, 438, 119, 33, 37, 33, 37, 22, 37, - /* 2380 */ 0, 22, 37, 0, 22, 0, 22, 436, 420, 438, - /* 2390 */ 0, 22, 0, 0, 0, 37, 0, 37, 0, 22, - /* 2400 */ 54, 20, 37, 37, 436, 37, 438, 37, 0, 0, - /* 2410 */ 379, 114, 49, 37, 0, 113, 0, 220, 484, 113, - /* 2420 */ 0, 487, 226, 392, 225, 491, 492, 493, 494, 495, - /* 2430 */ 496, 3, 498, 190, 22, 484, 33, 22, 487, 301, - /* 2440 */ 33, 37, 491, 492, 493, 494, 495, 496, 190, 498, - /* 2450 */ 190, 420, 484, 0, 37, 487, 50, 196, 111, 491, - /* 2460 */ 492, 493, 494, 495, 496, 33, 498, 436, 190, 438, - /* 2470 */ 33, 190, 216, 200, 200, 50, 49, 33, 49, 80, - /* 2480 */ 113, 109, 301, 33, 37, 114, 113, 3, 114, 114, - /* 2490 */ 379, 113, 33, 114, 37, 113, 0, 113, 37, 113, - /* 2500 */ 37, 114, 113, 392, 301, 114, 37, 379, 113, 37, - /* 2510 */ 37, 114, 33, 49, 114, 484, 114, 49, 487, 114, - /* 2520 */ 392, 0, 491, 492, 493, 494, 495, 496, 42, 498, - /* 2530 */ 42, 420, 113, 379, 0, 114, 42, 114, 114, 113, - /* 2540 */ 33, 193, 2, 113, 113, 111, 392, 436, 420, 438, - /* 2550 */ 22, 197, 113, 111, 285, 272, 249, 49, 49, 114, - /* 2560 */ 114, 113, 113, 22, 436, 113, 438, 113, 113, 113, - /* 2570 */ 379, 0, 114, 113, 420, 252, 114, 42, 113, 113, - /* 2580 */ 113, 193, 49, 392, 114, 114, 22, 113, 22, 113, - /* 2590 */ 436, 113, 438, 192, 113, 484, 113, 122, 487, 22, - /* 2600 */ 123, 114, 491, 492, 493, 494, 495, 496, 113, 498, - /* 2610 */ 37, 420, 484, 37, 114, 487, 37, 114, 113, 491, - /* 2620 */ 492, 493, 494, 495, 496, 37, 498, 436, 193, 438, - /* 2630 */ 114, 379, 37, 114, 37, 37, 226, 114, 484, 113, - /* 2640 */ 134, 487, 33, 113, 392, 491, 492, 493, 494, 495, - /* 2650 */ 496, 134, 498, 37, 379, 113, 134, 22, 134, 74, - /* 2660 */ 73, 22, 37, 37, 37, 37, 37, 392, 37, 37, - /* 2670 */ 37, 37, 420, 37, 37, 484, 80, 107, 487, 80, + /* 2220 */ 0, 22, 461, 154, 0, 153, 436, 0, 438, 152, + /* 2230 */ 0, 420, 392, 0, 0, 66, 0, 50, 66, 0, + /* 2240 */ 50, 22, 66, 22, 0, 484, 0, 436, 487, 438, + /* 2250 */ 0, 461, 491, 492, 493, 494, 495, 496, 0, 498, + /* 2260 */ 420, 0, 37, 66, 42, 37, 37, 0, 42, 37, + /* 2270 */ 0, 14, 0, 42, 484, 52, 436, 487, 438, 37, + /* 2280 */ 42, 491, 492, 493, 494, 495, 496, 52, 498, 52, + /* 2290 */ 45, 42, 33, 0, 0, 484, 49, 49, 487, 43, + /* 2300 */ 49, 0, 491, 492, 493, 494, 495, 496, 0, 498, + /* 2310 */ 42, 0, 204, 379, 49, 0, 0, 0, 49, 0, + /* 2320 */ 0, 37, 42, 52, 484, 0, 392, 487, 74, 37, + /* 2330 */ 379, 491, 492, 493, 494, 495, 496, 52, 498, 42, + /* 2340 */ 0, 37, 52, 392, 42, 0, 37, 379, 52, 42, + /* 2350 */ 0, 0, 0, 0, 420, 0, 0, 22, 37, 0, + /* 2360 */ 392, 121, 119, 22, 37, 37, 37, 37, 37, 37, + /* 2370 */ 436, 420, 438, 0, 37, 37, 37, 33, 0, 33, + /* 2380 */ 37, 0, 22, 0, 37, 37, 22, 436, 420, 438, + /* 2390 */ 22, 22, 22, 0, 37, 0, 54, 0, 0, 37, + /* 2400 */ 37, 0, 22, 37, 436, 20, 438, 0, 37, 37, + /* 2410 */ 379, 0, 114, 113, 49, 0, 113, 0, 484, 37, + /* 2420 */ 226, 487, 22, 392, 22, 491, 492, 493, 494, 495, + /* 2430 */ 496, 190, 498, 0, 3, 484, 33, 301, 487, 113, + /* 2440 */ 50, 37, 491, 492, 493, 494, 495, 496, 114, 498, + /* 2450 */ 190, 420, 484, 37, 190, 487, 190, 33, 196, 491, + /* 2460 */ 492, 493, 494, 495, 496, 111, 498, 436, 190, 438, + /* 2470 */ 50, 113, 33, 200, 200, 220, 49, 33, 225, 216, + /* 2480 */ 33, 114, 301, 49, 109, 80, 33, 113, 37, 113, + /* 2490 */ 379, 114, 3, 114, 33, 37, 37, 113, 49, 113, + /* 2500 */ 37, 114, 113, 392, 301, 114, 114, 379, 114, 113, + /* 2510 */ 37, 37, 37, 114, 33, 484, 49, 0, 487, 0, + /* 2520 */ 392, 113, 491, 492, 493, 494, 495, 496, 114, 498, + /* 2530 */ 42, 420, 114, 379, 0, 114, 113, 42, 113, 113, + /* 2540 */ 0, 113, 193, 114, 42, 33, 392, 436, 420, 438, + /* 2550 */ 2, 111, 111, 272, 22, 249, 114, 49, 49, 114, + /* 2560 */ 114, 285, 114, 113, 436, 113, 438, 113, 113, 113, + /* 2570 */ 379, 22, 0, 113, 420, 113, 252, 113, 113, 42, + /* 2580 */ 114, 113, 113, 392, 49, 197, 114, 37, 22, 113, + /* 2590 */ 436, 113, 438, 122, 114, 484, 113, 113, 487, 193, + /* 2600 */ 123, 192, 491, 492, 493, 494, 495, 496, 22, 498, + /* 2610 */ 37, 420, 484, 22, 113, 487, 37, 114, 113, 491, + /* 2620 */ 492, 493, 494, 495, 496, 193, 498, 436, 114, 438, + /* 2630 */ 37, 379, 114, 37, 114, 37, 226, 114, 484, 37, + /* 2640 */ 134, 487, 113, 33, 392, 491, 492, 493, 494, 495, + /* 2650 */ 496, 113, 498, 37, 379, 134, 134, 113, 134, 22, + /* 2660 */ 22, 74, 73, 37, 37, 37, 37, 392, 37, 37, + /* 2670 */ 37, 37, 420, 37, 80, 484, 37, 37, 487, 80, /* 2680 */ 33, 107, 491, 492, 493, 494, 495, 496, 436, 498, - /* 2690 */ 438, 37, 37, 37, 22, 420, 37, 37, 37, 80, - /* 2700 */ 37, 37, 37, 37, 37, 22, 37, 0, 37, 52, - /* 2710 */ 42, 436, 0, 438, 37, 379, 42, 0, 37, 0, - /* 2720 */ 42, 37, 52, 0, 42, 52, 37, 52, 392, 37, - /* 2730 */ 22, 0, 22, 33, 554, 22, 484, 0, 22, 487, - /* 2740 */ 379, 21, 21, 491, 492, 493, 494, 495, 496, 22, - /* 2750 */ 498, 554, 20, 392, 554, 554, 420, 554, 554, 484, + /* 2690 */ 438, 37, 107, 37, 37, 420, 22, 37, 37, 37, + /* 2700 */ 80, 37, 37, 37, 37, 37, 22, 37, 0, 37, + /* 2710 */ 52, 436, 0, 438, 37, 379, 0, 42, 42, 37, + /* 2720 */ 42, 52, 0, 52, 37, 52, 0, 42, 392, 37, + /* 2730 */ 0, 22, 37, 0, 33, 22, 484, 22, 21, 487, + /* 2740 */ 379, 22, 22, 491, 492, 493, 494, 495, 496, 21, + /* 2750 */ 498, 20, 554, 392, 554, 554, 420, 554, 554, 484, /* 2760 */ 554, 554, 487, 554, 554, 554, 491, 492, 493, 494, /* 2770 */ 495, 496, 436, 498, 438, 554, 554, 554, 554, 554, /* 2780 */ 554, 420, 554, 554, 554, 554, 554, 554, 554, 554, @@ -1284,9 +1287,9 @@ static const YYCODETYPE yy_lookahead[] = { /* 3590 */ 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, /* 3600 */ 376, 376, 376, 376, }; -#define YY_SHIFT_COUNT (962) +#define YY_SHIFT_COUNT (964) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2737) +#define YY_SHIFT_MAX (2733) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 520, 0, 259, 0, 519, 519, 519, 519, 519, 519, /* 10 */ 519, 519, 519, 519, 519, 519, 778, 1037, 1037, 1296, @@ -1326,67 +1329,67 @@ static const unsigned short int yy_shift_ofst[] = { /* 350 */ 503, 503, 503, 503, 503, 503, 503, 503, 503, 503, /* 360 */ 503, 503, 503, 503, 503, 503, 503, 503, 1904, 503, /* 370 */ 1701, 404, 1996, 1983, 1774, 3228, 3228, 3228, 3228, 3228, - /* 380 */ 3228, 3228, 3228, 3228, 3228, 3228, 81, 1425, 359, 1684, - /* 390 */ 391, 17, 558, 63, 707, 1492, 231, 724, 588, 588, - /* 400 */ 588, 588, 588, 588, 588, 588, 588, 750, 610, 464, - /* 410 */ 66, 723, 723, 402, 733, 39, 782, 578, 760, 846, - /* 420 */ 578, 889, 802, 1075, 54, 302, 302, 1231, 1113, 964, - /* 430 */ 1231, 1231, 1231, 1404, 1255, 1336, 605, 1151, 696, 1440, - /* 440 */ 1313, 1315, 1317, 1328, 734, 1441, 1442, 1476, 1480, 1490, - /* 450 */ 1493, 1226, 1416, 1420, 8, 1477, 1481, 1485, 1487, 1332, - /* 460 */ 1303, 621, 1483, 1496, 1499, 1541, 1502, 781, 1511, 1409, - /* 470 */ 1535, 1538, 1547, 1550, 1526, 1560, 1572, 1577, 1592, 1602, - /* 480 */ 1630, 1635, 1642, 1644, 1647, 1563, 1565, 1567, 1573, 1591, - /* 490 */ 1600, 429, 1446, 498, 1589, 1504, 1632, 1621, 1390, 2080, - /* 500 */ 2082, 2083, 2040, 2087, 2051, 1849, 2053, 2054, 2056, 1855, - /* 510 */ 2094, 2059, 2060, 1858, 2062, 2100, 2101, 1862, 2103, 2067, - /* 520 */ 2105, 2069, 2107, 2089, 2109, 2075, 1878, 2114, 1894, 2116, - /* 530 */ 1897, 1899, 1907, 1906, 2124, 2125, 2126, 1924, 1927, 2136, - /* 540 */ 2137, 1984, 2090, 2092, 2142, 2106, 2144, 2145, 2110, 2096, - /* 550 */ 2146, 2102, 2149, 2112, 2150, 2152, 2158, 2113, 2160, 2161, - /* 560 */ 2171, 2173, 2174, 2175, 1981, 2118, 2176, 2004, 2178, 2179, - /* 570 */ 2180, 2183, 2184, 2185, 2186, 2188, 2189, 2198, 2200, 2201, - /* 580 */ 2202, 2204, 2205, 2206, 2207, 2208, 2209, 2211, 2143, 2193, - /* 590 */ 2151, 2194, 2195, 2212, 2213, 2214, 2215, 2216, 2217, 2218, - /* 600 */ 2169, 2220, 2070, 2221, 2072, 2223, 2077, 2227, 2230, 2219, - /* 610 */ 2187, 2222, 2190, 2233, 2168, 2235, 2170, 2224, 2238, 2177, - /* 620 */ 2239, 2192, 2242, 2246, 2225, 2228, 2226, 2250, 2229, 2237, - /* 630 */ 2231, 2263, 2232, 2240, 2245, 2264, 2234, 2265, 2248, 2249, - /* 640 */ 2257, 2247, 2251, 2253, 2259, 2270, 2236, 2255, 2272, 2294, - /* 650 */ 2299, 2301, 2268, 2071, 2311, 2247, 2266, 2312, 2247, 2267, - /* 660 */ 2314, 2317, 2244, 2319, 2320, 2284, 2271, 2280, 2325, 2291, - /* 670 */ 2277, 2295, 2339, 2303, 2289, 2300, 2344, 2308, 2296, 2304, - /* 680 */ 2349, 2350, 2351, 2352, 2353, 2355, 2241, 2254, 2321, 2334, - /* 690 */ 2357, 2337, 2324, 2326, 2327, 2328, 2329, 2330, 2331, 2332, - /* 700 */ 2338, 2341, 2343, 2340, 2342, 2356, 2345, 2380, 2359, 2383, - /* 710 */ 2362, 2385, 2364, 2346, 2390, 2369, 2358, 2392, 2393, 2394, - /* 720 */ 2360, 2396, 2365, 2398, 2377, 2381, 2366, 2368, 2370, 2297, - /* 730 */ 2302, 2408, 2243, 2196, 2199, 2306, 2197, 2247, 2363, 2409, - /* 740 */ 2258, 2376, 2412, 2414, 2256, 2415, 2260, 2261, 2416, 2420, - /* 750 */ 2278, 2273, 2281, 2274, 2428, 2403, 2138, 2367, 2371, 2373, - /* 760 */ 2374, 2404, 2417, 2378, 2406, 2347, 2425, 2372, 2375, 2407, - /* 770 */ 2432, 2379, 2382, 2384, 2386, 2387, 2437, 2427, 2429, 2389, - /* 780 */ 2444, 2181, 2399, 2391, 2450, 2395, 2447, 2397, 2400, 2484, - /* 790 */ 2459, 2203, 2457, 2461, 2463, 2469, 2472, 2473, 2402, 2405, - /* 800 */ 2464, 2269, 2479, 2468, 2453, 2496, 2419, 2486, 2421, 2423, - /* 810 */ 2426, 2430, 2348, 2431, 2521, 2488, 2354, 2534, 2424, 2439, - /* 820 */ 2388, 2494, 2401, 2507, 2434, 2283, 2442, 2540, 2528, 2307, - /* 830 */ 2445, 2446, 2448, 2449, 2452, 2454, 2455, 2458, 2508, 2456, - /* 840 */ 2460, 2509, 2462, 2541, 2323, 2465, 2466, 2571, 2470, 2467, - /* 850 */ 2435, 2535, 2474, 2475, 2247, 2533, 2476, 2478, 2471, 2481, - /* 860 */ 2483, 2477, 2564, 2566, 2577, 2410, 2487, 2573, 2576, 2495, - /* 870 */ 2500, 2579, 2505, 2503, 2588, 2448, 2516, 2595, 2449, 2519, - /* 880 */ 2597, 2452, 2523, 2598, 2454, 2506, 2517, 2522, 2524, 2526, - /* 890 */ 2609, 2530, 2616, 2542, 2609, 2609, 2635, 2585, 2587, 2639, - /* 900 */ 2625, 2626, 2627, 2628, 2629, 2631, 2632, 2633, 2634, 2636, - /* 910 */ 2637, 2596, 2570, 2599, 2574, 2647, 2654, 2655, 2656, 2672, - /* 920 */ 2659, 2660, 2661, 2619, 2341, 2663, 2343, 2664, 2665, 2666, - /* 930 */ 2667, 2683, 2669, 2707, 2671, 2657, 2668, 2712, 2677, 2670, - /* 940 */ 2674, 2717, 2681, 2673, 2678, 2719, 2684, 2675, 2682, 2723, - /* 950 */ 2689, 2737, 2708, 2692, 2731, 2710, 2700, 2713, 2720, 2716, - /* 960 */ 2727, 2721, 2732, + /* 380 */ 3228, 3228, 3228, 3228, 3228, 3228, 3228, 81, 1425, 359, + /* 390 */ 1684, 391, 17, 558, 63, 707, 1492, 231, 724, 588, + /* 400 */ 588, 588, 588, 588, 588, 588, 588, 588, 750, 610, + /* 410 */ 464, 66, 723, 723, 402, 733, 39, 782, 578, 578, + /* 420 */ 760, 846, 578, 889, 802, 1075, 54, 302, 302, 1231, + /* 430 */ 1113, 964, 1231, 1231, 1231, 1404, 1255, 1336, 605, 1151, + /* 440 */ 696, 1440, 1313, 1315, 1317, 1328, 734, 1441, 1442, 1476, + /* 450 */ 1480, 1490, 1493, 1226, 1416, 1420, 8, 1477, 1481, 1485, + /* 460 */ 1487, 1332, 1303, 621, 1483, 1496, 1499, 1541, 1502, 781, + /* 470 */ 1511, 1409, 1535, 1538, 1547, 1550, 1526, 1560, 1572, 1577, + /* 480 */ 1592, 1602, 1630, 1635, 1642, 1644, 1647, 1563, 1565, 1567, + /* 490 */ 1573, 1591, 1600, 429, 1446, 498, 1589, 1504, 1632, 1621, + /* 500 */ 1390, 2082, 2083, 2085, 2041, 2087, 2051, 1850, 2054, 2056, + /* 510 */ 2057, 1855, 2096, 2060, 2061, 1859, 2063, 2101, 2102, 1863, + /* 520 */ 2104, 2068, 2106, 2070, 2109, 2089, 2112, 2076, 1879, 2115, + /* 530 */ 1895, 2118, 1898, 1899, 1907, 1911, 2125, 2126, 2133, 1926, + /* 540 */ 1928, 2137, 2138, 1985, 2092, 2093, 2143, 2107, 2145, 2146, + /* 550 */ 2110, 2097, 2148, 2103, 2150, 2113, 2151, 2157, 2160, 2105, + /* 560 */ 2155, 2161, 2162, 2171, 2173, 2174, 2002, 2139, 2177, 2005, + /* 570 */ 2179, 2180, 2183, 2184, 2185, 2186, 2188, 2189, 2198, 2200, + /* 580 */ 2201, 2202, 2204, 2205, 2206, 2207, 2208, 2209, 2211, 2212, + /* 590 */ 2142, 2192, 2149, 2193, 2195, 2196, 2213, 2214, 2215, 2216, + /* 600 */ 2217, 2218, 2199, 2220, 2069, 2224, 2072, 2227, 2077, 2230, + /* 610 */ 2233, 2219, 2187, 2221, 2190, 2234, 2169, 2236, 2172, 2225, + /* 620 */ 2239, 2176, 2244, 2197, 2246, 2250, 2228, 2223, 2222, 2258, + /* 630 */ 2229, 2235, 2226, 2261, 2232, 2237, 2231, 2267, 2242, 2270, + /* 640 */ 2245, 2238, 2259, 2247, 2248, 2257, 2251, 2272, 2256, 2249, + /* 650 */ 2293, 2294, 2301, 2308, 2268, 2108, 2311, 2247, 2265, 2315, + /* 660 */ 2247, 2269, 2316, 2317, 2254, 2319, 2320, 2284, 2271, 2280, + /* 670 */ 2325, 2292, 2285, 2297, 2340, 2304, 2290, 2302, 2345, 2309, + /* 680 */ 2296, 2307, 2350, 2351, 2352, 2353, 2355, 2356, 2240, 2243, + /* 690 */ 2321, 2335, 2359, 2341, 2327, 2328, 2329, 2330, 2331, 2332, + /* 700 */ 2337, 2338, 2339, 2344, 2346, 2343, 2347, 2360, 2348, 2373, + /* 710 */ 2364, 2378, 2368, 2381, 2369, 2342, 2383, 2370, 2357, 2393, + /* 720 */ 2395, 2397, 2362, 2398, 2363, 2401, 2380, 2385, 2366, 2371, + /* 730 */ 2372, 2298, 2300, 2407, 2241, 2194, 2253, 2303, 2255, 2247, + /* 740 */ 2365, 2411, 2260, 2382, 2400, 2415, 2263, 2402, 2264, 2262, + /* 750 */ 2417, 2433, 2266, 2273, 2278, 2274, 2431, 2403, 2136, 2326, + /* 760 */ 2334, 2358, 2367, 2404, 2416, 2374, 2390, 2354, 2420, 2375, + /* 770 */ 2377, 2424, 2439, 2379, 2376, 2384, 2386, 2387, 2444, 2427, + /* 780 */ 2434, 2389, 2447, 2181, 2405, 2391, 2453, 2396, 2451, 2392, + /* 790 */ 2394, 2489, 2461, 2203, 2458, 2459, 2463, 2473, 2474, 2475, + /* 800 */ 2399, 2414, 2449, 2276, 2481, 2467, 2517, 2519, 2408, 2488, + /* 810 */ 2418, 2421, 2423, 2425, 2349, 2426, 2534, 2495, 2388, 2540, + /* 820 */ 2429, 2428, 2406, 2502, 2409, 2512, 2440, 2281, 2441, 2548, + /* 830 */ 2532, 2306, 2442, 2445, 2450, 2452, 2454, 2455, 2456, 2446, + /* 840 */ 2508, 2460, 2462, 2509, 2448, 2549, 2324, 2464, 2465, 2572, + /* 850 */ 2466, 2468, 2432, 2537, 2469, 2471, 2247, 2535, 2476, 2478, + /* 860 */ 2472, 2483, 2484, 2477, 2566, 2586, 2591, 2410, 2480, 2550, + /* 870 */ 2573, 2501, 2503, 2579, 2505, 2514, 2593, 2450, 2518, 2596, + /* 880 */ 2452, 2520, 2598, 2454, 2523, 2602, 2455, 2506, 2521, 2522, + /* 890 */ 2524, 2529, 2610, 2538, 2616, 2544, 2610, 2610, 2637, 2587, + /* 900 */ 2589, 2638, 2626, 2627, 2628, 2629, 2631, 2632, 2633, 2634, + /* 910 */ 2636, 2639, 2640, 2594, 2574, 2599, 2585, 2647, 2654, 2656, + /* 920 */ 2657, 2674, 2660, 2661, 2662, 2620, 2344, 2664, 2346, 2665, + /* 930 */ 2666, 2667, 2668, 2684, 2670, 2708, 2672, 2658, 2675, 2712, + /* 940 */ 2677, 2669, 2676, 2716, 2682, 2671, 2678, 2722, 2687, 2673, + /* 950 */ 2685, 2726, 2692, 2730, 2709, 2695, 2733, 2713, 2701, 2715, + /* 960 */ 2717, 2719, 2720, 2728, 2731, }; -#define YY_REDUCE_COUNT (385) +#define YY_REDUCE_COUNT (386) #define YY_REDUCE_MIN (-510) #define YY_REDUCE_MAX (2729) static const short yy_reduce_ofst[] = { @@ -1427,107 +1430,107 @@ static const short yy_reduce_ofst[] = { /* 340 */ 1555, 1559, 1610, 1594, 1614, 1622, 1580, 1583, 1584, 1587, /* 350 */ 1588, 1590, 1593, 1595, 1596, 1598, 1599, 1607, 1609, 1612, /* 360 */ 1613, 1617, 1619, 1624, 1625, 1627, 1629, 1640, 1597, 1641, - /* 370 */ 1634, 1638, 1646, 1662, 1668, 1579, 1571, 1574, 1626, 1628, - /* 380 */ 1643, 1664, 1653, 1665, 1659, 1704, + /* 370 */ 1634, 1638, 1646, 1662, 1668, 1579, 1628, 1571, 1604, 1626, + /* 380 */ 1631, 1643, 1664, 1653, 1665, 1659, 1704, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 10 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 20 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 30 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 40 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 50 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 60 */ 2547, 2179, 2179, 2503, 2179, 2179, 2179, 2179, 2179, 2179, - /* 70 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2510, 2179, - /* 80 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 90 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2280, - /* 100 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 110 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2278, 2799, - /* 120 */ 2179, 2925, 2588, 2179, 2179, 2828, 2179, 2179, 2179, 2179, - /* 130 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 140 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2811, 2179, 2179, - /* 150 */ 2251, 2251, 2179, 2811, 2811, 2811, 2771, 2771, 2278, 2179, - /* 160 */ 2179, 2280, 2179, 2590, 2179, 2179, 2179, 2179, 2179, 2179, - /* 170 */ 2179, 2179, 2179, 2420, 2209, 2179, 2179, 2179, 2179, 2179, - /* 180 */ 2179, 2573, 2179, 2179, 2857, 2803, 2804, 2919, 2179, 2860, - /* 190 */ 2822, 2179, 2817, 2179, 2179, 2179, 2179, 2515, 2179, 2847, - /* 200 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2616, 2179, 2179, - /* 210 */ 2365, 2567, 2179, 2179, 2179, 2179, 2179, 2903, 2801, 2841, - /* 220 */ 2179, 2851, 2179, 2179, 2179, 2604, 2280, 2179, 2280, 2560, - /* 230 */ 2498, 2179, 2508, 2179, 2508, 2505, 2179, 2179, 2179, 2508, - /* 240 */ 2505, 2505, 2505, 2354, 2350, 2179, 2179, 2348, 2179, 2179, - /* 250 */ 2179, 2179, 2234, 2179, 2234, 2179, 2280, 2280, 2179, 2280, - /* 260 */ 2179, 2179, 2280, 2179, 2280, 2179, 2280, 2179, 2280, 2280, - /* 270 */ 2179, 2280, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 280 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 290 */ 2179, 2602, 2583, 2179, 2278, 2179, 2571, 2569, 2179, 2278, - /* 300 */ 2851, 2179, 2179, 2873, 2868, 2873, 2868, 2887, 2883, 2873, - /* 310 */ 2892, 2889, 2853, 2851, 2834, 2830, 2922, 2909, 2905, 2179, - /* 320 */ 2179, 2839, 2837, 2179, 2278, 2278, 2868, 2179, 2179, 2179, - /* 330 */ 2179, 2868, 2179, 2179, 2278, 2179, 2179, 2278, 2179, 2179, - /* 340 */ 2179, 2179, 2278, 2179, 2278, 2179, 2179, 2179, 2179, 2179, - /* 350 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 360 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2382, 2179, - /* 370 */ 2179, 2278, 2179, 2218, 2179, 2562, 2588, 2593, 2543, 2543, - /* 380 */ 2423, 2423, 2925, 2423, 2281, 2184, 2179, 2179, 2179, 2179, - /* 390 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2886, 2885, - /* 400 */ 2722, 2179, 2775, 2774, 2773, 2764, 2721, 2378, 2179, 2179, - /* 410 */ 2179, 2720, 2719, 2179, 2179, 2179, 2179, 2369, 2179, 2179, - /* 420 */ 2391, 2179, 2179, 2179, 2179, 2534, 2533, 2713, 2179, 2179, - /* 430 */ 2714, 2712, 2711, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 440 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 450 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 460 */ 2906, 2910, 2179, 2179, 2179, 2800, 2179, 2179, 2179, 2692, - /* 470 */ 2179, 2179, 2179, 2179, 2660, 2655, 2646, 2637, 2652, 2643, - /* 480 */ 2631, 2649, 2640, 2628, 2625, 2179, 2179, 2179, 2179, 2179, - /* 490 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 500 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 510 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 520 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 530 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 540 */ 2179, 2504, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 550 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 560 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 570 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 580 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 590 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 600 */ 2179, 2179, 2179, 2179, 2179, 2179, 2519, 2179, 2179, 2179, - /* 610 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 620 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 630 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 640 */ 2223, 2699, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 650 */ 2179, 2179, 2179, 2179, 2179, 2702, 2179, 2179, 2703, 2179, - /* 660 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 670 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 680 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 690 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 700 */ 2179, 2325, 2324, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 710 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 720 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2704, - /* 730 */ 2179, 2179, 2179, 2179, 2587, 2179, 2179, 2694, 2179, 2179, - /* 740 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 750 */ 2179, 2179, 2179, 2179, 2902, 2854, 2179, 2179, 2179, 2179, - /* 760 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 770 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2692, 2179, - /* 780 */ 2884, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2900, 2179, - /* 790 */ 2904, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2810, 2806, - /* 800 */ 2179, 2179, 2802, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 810 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 820 */ 2179, 2179, 2179, 2761, 2179, 2179, 2179, 2795, 2179, 2179, - /* 830 */ 2179, 2179, 2419, 2418, 2417, 2416, 2179, 2179, 2179, 2179, - /* 840 */ 2179, 2179, 2704, 2179, 2707, 2179, 2179, 2179, 2179, 2179, - /* 850 */ 2179, 2179, 2179, 2179, 2691, 2179, 2746, 2745, 2179, 2179, - /* 860 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2413, - /* 870 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 880 */ 2179, 2179, 2179, 2179, 2179, 2397, 2395, 2394, 2393, 2179, - /* 890 */ 2430, 2179, 2179, 2179, 2426, 2425, 2179, 2179, 2179, 2179, - /* 900 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 910 */ 2179, 2179, 2179, 2179, 2179, 2299, 2179, 2179, 2179, 2179, - /* 920 */ 2179, 2179, 2179, 2179, 2291, 2179, 2290, 2179, 2179, 2179, - /* 930 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 940 */ 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, 2179, - /* 950 */ 2179, 2179, 2179, 2179, 2179, 2179, 2208, 2179, 2179, 2179, - /* 960 */ 2179, 2179, 2179, + /* 0 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 10 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 20 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 30 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 40 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 50 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 60 */ 2548, 2180, 2180, 2504, 2180, 2180, 2180, 2180, 2180, 2180, + /* 70 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2511, 2180, + /* 80 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 90 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2281, + /* 100 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 110 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2279, 2800, + /* 120 */ 2180, 2926, 2589, 2180, 2180, 2829, 2180, 2180, 2180, 2180, + /* 130 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 140 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2812, 2180, 2180, + /* 150 */ 2252, 2252, 2180, 2812, 2812, 2812, 2772, 2772, 2279, 2180, + /* 160 */ 2180, 2281, 2180, 2591, 2180, 2180, 2180, 2180, 2180, 2180, + /* 170 */ 2180, 2180, 2180, 2421, 2210, 2180, 2180, 2180, 2180, 2180, + /* 180 */ 2180, 2574, 2180, 2180, 2858, 2804, 2805, 2920, 2180, 2861, + /* 190 */ 2823, 2180, 2818, 2180, 2180, 2180, 2180, 2516, 2180, 2848, + /* 200 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2617, 2180, 2180, + /* 210 */ 2366, 2568, 2180, 2180, 2180, 2180, 2180, 2904, 2802, 2842, + /* 220 */ 2180, 2852, 2180, 2180, 2180, 2605, 2281, 2180, 2281, 2561, + /* 230 */ 2499, 2180, 2509, 2180, 2509, 2506, 2180, 2180, 2180, 2509, + /* 240 */ 2506, 2506, 2506, 2355, 2351, 2180, 2180, 2349, 2180, 2180, + /* 250 */ 2180, 2180, 2235, 2180, 2235, 2180, 2281, 2281, 2180, 2281, + /* 260 */ 2180, 2180, 2281, 2180, 2281, 2180, 2281, 2180, 2281, 2281, + /* 270 */ 2180, 2281, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 280 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 290 */ 2180, 2603, 2584, 2180, 2279, 2180, 2572, 2570, 2180, 2279, + /* 300 */ 2852, 2180, 2180, 2874, 2869, 2874, 2869, 2888, 2884, 2874, + /* 310 */ 2893, 2890, 2854, 2852, 2835, 2831, 2923, 2910, 2906, 2180, + /* 320 */ 2180, 2840, 2838, 2180, 2279, 2279, 2869, 2180, 2180, 2180, + /* 330 */ 2180, 2869, 2180, 2180, 2279, 2180, 2180, 2279, 2180, 2180, + /* 340 */ 2180, 2180, 2279, 2180, 2279, 2180, 2180, 2180, 2180, 2180, + /* 350 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 360 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2383, 2180, + /* 370 */ 2180, 2279, 2180, 2219, 2180, 2563, 2926, 2589, 2594, 2544, + /* 380 */ 2544, 2424, 2424, 2926, 2424, 2282, 2185, 2180, 2180, 2180, + /* 390 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2887, + /* 400 */ 2886, 2723, 2180, 2776, 2775, 2774, 2765, 2722, 2379, 2180, + /* 410 */ 2180, 2180, 2721, 2720, 2180, 2180, 2180, 2180, 2370, 2367, + /* 420 */ 2180, 2180, 2392, 2180, 2180, 2180, 2180, 2535, 2534, 2714, + /* 430 */ 2180, 2180, 2715, 2713, 2712, 2180, 2180, 2180, 2180, 2180, + /* 440 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 450 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 460 */ 2180, 2180, 2907, 2911, 2180, 2180, 2180, 2801, 2180, 2180, + /* 470 */ 2180, 2693, 2180, 2180, 2180, 2180, 2661, 2656, 2647, 2638, + /* 480 */ 2653, 2644, 2632, 2650, 2641, 2629, 2626, 2180, 2180, 2180, + /* 490 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 500 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 510 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 520 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 530 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 540 */ 2180, 2180, 2180, 2505, 2180, 2180, 2180, 2180, 2180, 2180, + /* 550 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 560 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 570 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 580 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 590 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 600 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2520, 2180, + /* 610 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 620 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 630 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 640 */ 2180, 2180, 2224, 2700, 2180, 2180, 2180, 2180, 2180, 2180, + /* 650 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2703, 2180, 2180, + /* 660 */ 2704, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 670 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 680 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 690 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 700 */ 2180, 2180, 2180, 2326, 2325, 2180, 2180, 2180, 2180, 2180, + /* 710 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 720 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 730 */ 2180, 2705, 2180, 2180, 2180, 2180, 2588, 2180, 2180, 2695, + /* 740 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 750 */ 2180, 2180, 2180, 2180, 2180, 2180, 2903, 2855, 2180, 2180, + /* 760 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 770 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 780 */ 2693, 2180, 2885, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 790 */ 2901, 2180, 2905, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 800 */ 2811, 2807, 2180, 2180, 2803, 2180, 2180, 2180, 2180, 2180, + /* 810 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 820 */ 2180, 2180, 2180, 2180, 2180, 2762, 2180, 2180, 2180, 2796, + /* 830 */ 2180, 2180, 2180, 2180, 2420, 2419, 2418, 2417, 2180, 2180, + /* 840 */ 2180, 2180, 2180, 2180, 2705, 2180, 2708, 2180, 2180, 2180, + /* 850 */ 2180, 2180, 2180, 2180, 2180, 2180, 2692, 2180, 2747, 2746, + /* 860 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 870 */ 2180, 2414, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 880 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2398, 2396, 2395, + /* 890 */ 2394, 2180, 2431, 2180, 2180, 2180, 2427, 2426, 2180, 2180, + /* 900 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 910 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2300, 2180, 2180, + /* 920 */ 2180, 2180, 2180, 2180, 2180, 2180, 2292, 2180, 2291, 2180, + /* 930 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 940 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, + /* 950 */ 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2180, 2209, 2180, + /* 960 */ 2180, 2180, 2180, 2180, 2180, }; /********** End of lemon-generated parsing tables *****************************/ @@ -2756,7 +2759,7 @@ static const char *const yyRuleName[] = { /* 181 */ "cmd ::= ALTER TABLE alter_table_clause", /* 182 */ "cmd ::= ALTER STABLE alter_table_clause", /* 183 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 184 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 184 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options", /* 185 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", /* 186 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", /* 187 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options", @@ -3533,7 +3536,9 @@ static void yy_destructor( case 549: /* query_simple_or_subquery */ case 551: /* sort_specification */ { +#line 7 "sql.y" nodesDestroyNode((yypminor->yy600)); +#line 3541 "sql.c" } break; case 377: /* account_options */ @@ -3543,7 +3548,9 @@ static void yy_destructor( case 462: /* with_meta */ case 471: /* bufsize_opt */ { +#line 54 "sql.y" +#line 3553 "sql.c" } break; case 381: /* ip_range_list */ @@ -3583,7 +3590,9 @@ static void yy_destructor( case 545: /* order_by_clause_opt */ case 550: /* sort_specification_list */ { +#line 85 "sql.y" nodesDestroyList((yypminor->yy748)); +#line 3595 "sql.c" } break; case 384: /* user_name */ @@ -3607,24 +3616,32 @@ static void yy_destructor( case 498: /* noarg_func */ case 516: /* alias_opt */ { +#line 1069 "sql.y" +#line 3621 "sql.c" } break; case 385: /* sysinfo_opt */ { +#line 112 "sql.y" +#line 3628 "sql.c" } break; case 386: /* privileges */ case 389: /* priv_type_list */ case 390: /* priv_type */ { +#line 121 "sql.y" +#line 3637 "sql.c" } break; case 387: /* priv_level */ { +#line 138 "sql.y" +#line 3644 "sql.c" } break; case 396: /* force_opt */ @@ -3638,66 +3655,90 @@ static void yy_destructor( case 527: /* set_quantifier_opt */ case 528: /* tag_mode_opt */ { +#line 170 "sql.y" +#line 3660 "sql.c" } break; case 409: /* alter_db_option */ case 435: /* alter_table_option */ { +#line 278 "sql.y" +#line 3668 "sql.c" } break; case 421: /* type_name */ case 432: /* type_name_default_len */ { +#line 413 "sql.y" +#line 3676 "sql.c" } break; case 440: /* db_kind_opt */ case 447: /* table_kind */ { +#line 591 "sql.y" +#line 3684 "sql.c" } break; case 441: /* table_kind_db_name_cond_opt */ { +#line 556 "sql.y" +#line 3691 "sql.c" } break; case 451: /* tsma_func_list */ { +#line 610 "sql.y" nodesDestroyNode((yypminor->yy600)); +#line 3698 "sql.c" } break; case 506: /* compare_op */ case 507: /* in_op */ { +#line 1267 "sql.y" +#line 3706 "sql.c" } break; case 519: /* join_type */ { +#line 1348 "sql.y" +#line 3713 "sql.c" } break; case 520: /* join_subtype */ { +#line 1356 "sql.y" +#line 3720 "sql.c" } break; case 541: /* fill_mode */ { +#line 1472 "sql.y" +#line 3727 "sql.c" } break; case 552: /* ordering_specification_opt */ { +#line 1557 "sql.y" +#line 3734 "sql.c" } break; case 553: /* null_ordering_opt */ { +#line 1563 "sql.y" +#line 3741 "sql.c" } break; /********* End destructor definitions *****************************************/ @@ -4170,7 +4211,7 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 376, /* (181) cmd ::= ALTER TABLE alter_table_clause */ 376, /* (182) cmd ::= ALTER STABLE alter_table_clause */ 418, /* (183) alter_table_clause ::= full_table_name alter_table_options */ - 418, /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + 418, /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */ 418, /* (185) alter_table_clause ::= full_table_name DROP COLUMN column_name */ 418, /* (186) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ 418, /* (187) alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */ @@ -4923,7 +4964,7 @@ static const signed char yyRuleInfoNRhs[] = { -3, /* (181) cmd ::= ALTER TABLE alter_table_clause */ -3, /* (182) cmd ::= ALTER STABLE alter_table_clause */ -2, /* (183) alter_table_clause ::= full_table_name alter_table_options */ - -5, /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + -6, /* (184) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */ -4, /* (185) alter_table_clause ::= full_table_name DROP COLUMN column_name */ -5, /* (186) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -5, /* (187) alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */ @@ -5529,15 +5570,21 @@ static YYACTIONTYPE yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ +#line 50 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +#line 5575 "sql.c" yy_destructor(yypParser,377,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ +#line 51 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +#line 5581 "sql.c" yy_destructor(yypParser,378,&yymsp[0].minor); break; case 2: /* account_options ::= */ +#line 55 "sql.y" { } +#line 5587 "sql.c" break; case 3: /* account_options ::= account_options PPS literal */ case 4: /* account_options ::= account_options TSERIES literal */ yytestcase(yyruleno==4); @@ -5549,18 +5596,24 @@ static YYACTIONTYPE yy_reduce( case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); { yy_destructor(yypParser,377,&yymsp[-2].minor); +#line 56 "sql.y" { } +#line 5601 "sql.c" yy_destructor(yypParser,379,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ { yy_destructor(yypParser,380,&yymsp[0].minor); +#line 68 "sql.y" { } +#line 5609 "sql.c" } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ { yy_destructor(yypParser,378,&yymsp[-1].minor); +#line 69 "sql.y" { } +#line 5616 "sql.c" yy_destructor(yypParser,380,&yymsp[0].minor); } break; @@ -5574,19 +5627,27 @@ static YYACTIONTYPE yy_reduce( case 21: /* alter_account_option ::= USERS literal */ yytestcase(yyruleno==21); case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); +#line 73 "sql.y" { } +#line 5632 "sql.c" yy_destructor(yypParser,379,&yymsp[0].minor); break; case 24: /* ip_range_list ::= NK_STRING */ +#line 86 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 5638 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 25: /* ip_range_list ::= ip_range_list NK_COMMA NK_STRING */ +#line 87 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 5644 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 26: /* white_list ::= HOST ip_range_list */ +#line 91 "sql.y" { yymsp[-1].minor.yy748 = yymsp[0].minor.yy748; } +#line 5650 "sql.c" break; case 27: /* white_list_opt ::= */ case 200: /* specific_cols_opt ::= */ yytestcase(yyruleno==200); @@ -5597,89 +5658,137 @@ static YYACTIONTYPE yy_reduce( case 671: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==671); case 701: /* group_by_clause_opt ::= */ yytestcase(yyruleno==701); case 721: /* order_by_clause_opt ::= */ yytestcase(yyruleno==721); +#line 95 "sql.y" { yymsp[1].minor.yy748 = NULL; } +#line 5663 "sql.c" break; case 28: /* white_list_opt ::= white_list */ case 239: /* tags_def_opt ::= tags_def */ yytestcase(yyruleno==239); case 409: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==409); case 579: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==579); +#line 96 "sql.y" { yylhsminor.yy748 = yymsp[0].minor.yy748; } +#line 5671 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 29: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt white_list_opt */ +#line 100 "sql.y" { pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-4].minor.yy649, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy663); pCxt->pRootNode = addCreateUserStmtWhiteList(pCxt, pCxt->pRootNode, yymsp[0].minor.yy748); } +#line 5680 "sql.c" break; case 30: /* cmd ::= ALTER USER user_name PASS NK_STRING */ +#line 104 "sql.y" { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +#line 5685 "sql.c" break; case 31: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ +#line 105 "sql.y" { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +#line 5690 "sql.c" break; case 32: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ +#line 106 "sql.y" { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +#line 5695 "sql.c" break; case 33: /* cmd ::= ALTER USER user_name ADD white_list */ +#line 107 "sql.y" { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_ADD_WHITE_LIST, yymsp[0].minor.yy748); } +#line 5700 "sql.c" break; case 34: /* cmd ::= ALTER USER user_name DROP white_list */ +#line 108 "sql.y" { pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy649, TSDB_ALTER_USER_DROP_WHITE_LIST, yymsp[0].minor.yy748); } +#line 5705 "sql.c" break; case 35: /* cmd ::= DROP USER user_name */ +#line 109 "sql.y" { pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy649); } +#line 5710 "sql.c" break; case 36: /* sysinfo_opt ::= */ +#line 113 "sql.y" { yymsp[1].minor.yy663 = 1; } +#line 5715 "sql.c" break; case 37: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ +#line 114 "sql.y" { yymsp[-1].minor.yy663 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +#line 5720 "sql.c" break; case 38: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ +#line 117 "sql.y" { pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy941, &yymsp[-3].minor.yy781, &yymsp[0].minor.yy649, yymsp[-2].minor.yy600); } +#line 5725 "sql.c" break; case 39: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ +#line 118 "sql.y" { pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy941, &yymsp[-3].minor.yy781, &yymsp[0].minor.yy649, yymsp[-2].minor.yy600); } +#line 5730 "sql.c" break; case 40: /* privileges ::= ALL */ +#line 122 "sql.y" { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_ALL; } +#line 5735 "sql.c" break; case 41: /* privileges ::= priv_type_list */ case 43: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==43); +#line 123 "sql.y" { yylhsminor.yy941 = yymsp[0].minor.yy941; } +#line 5741 "sql.c" yymsp[0].minor.yy941 = yylhsminor.yy941; break; case 42: /* privileges ::= SUBSCRIBE */ +#line 124 "sql.y" { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_SUBSCRIBE; } +#line 5747 "sql.c" break; case 44: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ +#line 129 "sql.y" { yylhsminor.yy941 = yymsp[-2].minor.yy941 | yymsp[0].minor.yy941; } +#line 5752 "sql.c" yymsp[-2].minor.yy941 = yylhsminor.yy941; break; case 45: /* priv_type ::= READ */ +#line 133 "sql.y" { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_READ; } +#line 5758 "sql.c" break; case 46: /* priv_type ::= WRITE */ +#line 134 "sql.y" { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_WRITE; } +#line 5763 "sql.c" break; case 47: /* priv_type ::= ALTER */ +#line 135 "sql.y" { yymsp[0].minor.yy941 = PRIVILEGE_TYPE_ALTER; } +#line 5768 "sql.c" break; case 48: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ +#line 139 "sql.y" { yylhsminor.yy781.first = yymsp[-2].minor.yy0; yylhsminor.yy781.second = yymsp[0].minor.yy0; } +#line 5773 "sql.c" yymsp[-2].minor.yy781 = yylhsminor.yy781; break; case 49: /* priv_level ::= db_name NK_DOT NK_STAR */ +#line 140 "sql.y" { yylhsminor.yy781.first = yymsp[-2].minor.yy649; yylhsminor.yy781.second = yymsp[0].minor.yy0; } +#line 5779 "sql.c" yymsp[-2].minor.yy781 = yylhsminor.yy781; break; case 50: /* priv_level ::= db_name NK_DOT table_name */ +#line 141 "sql.y" { yylhsminor.yy781.first = yymsp[-2].minor.yy649; yylhsminor.yy781.second = yymsp[0].minor.yy649; } +#line 5785 "sql.c" yymsp[-2].minor.yy781 = yylhsminor.yy781; break; case 51: /* priv_level ::= topic_name */ +#line 142 "sql.y" { yylhsminor.yy781.first = yymsp[0].minor.yy649; yylhsminor.yy781.second = nil_token; } +#line 5791 "sql.c" yymsp[0].minor.yy781 = yylhsminor.yy781; break; case 52: /* with_opt ::= */ @@ -5701,50 +5810,78 @@ static YYACTIONTYPE yy_reduce( case 710: /* every_opt ::= */ yytestcase(yyruleno==710); case 723: /* slimit_clause_opt ::= */ yytestcase(yyruleno==723); case 727: /* limit_clause_opt ::= */ yytestcase(yyruleno==727); +#line 144 "sql.y" { yymsp[1].minor.yy600 = NULL; } +#line 5815 "sql.c" break; case 53: /* with_opt ::= WITH search_condition */ case 620: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==620); case 647: /* join_on_clause_opt ::= ON search_condition */ yytestcase(yyruleno==647); case 670: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==670); case 706: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==706); +#line 145 "sql.y" { yymsp[-1].minor.yy600 = yymsp[0].minor.yy600; } +#line 5824 "sql.c" break; case 54: /* cmd ::= CREATE ENCRYPT_KEY NK_STRING */ +#line 148 "sql.y" { pCxt->pRootNode = createEncryptKeyStmt(pCxt, &yymsp[0].minor.yy0); } +#line 5829 "sql.c" break; case 55: /* cmd ::= CREATE DNODE dnode_endpoint */ +#line 151 "sql.y" { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy649, NULL); } +#line 5834 "sql.c" break; case 56: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ +#line 152 "sql.y" { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0); } +#line 5839 "sql.c" break; case 57: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ +#line 153 "sql.y" { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy705, false); } +#line 5844 "sql.c" break; case 58: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ +#line 154 "sql.y" { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy705, false); } +#line 5849 "sql.c" break; case 59: /* cmd ::= DROP DNODE NK_INTEGER unsafe_opt */ +#line 155 "sql.y" { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, false, yymsp[0].minor.yy705); } +#line 5854 "sql.c" break; case 60: /* cmd ::= DROP DNODE dnode_endpoint unsafe_opt */ +#line 156 "sql.y" { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy649, false, yymsp[0].minor.yy705); } +#line 5859 "sql.c" break; case 61: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ +#line 157 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } +#line 5864 "sql.c" break; case 62: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ +#line 158 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } +#line 5869 "sql.c" break; case 63: /* cmd ::= ALTER ALL DNODES NK_STRING */ +#line 159 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } +#line 5874 "sql.c" break; case 64: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ +#line 160 "sql.y" { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } +#line 5879 "sql.c" break; case 65: /* cmd ::= RESTORE DNODE NK_INTEGER */ +#line 161 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_DNODE_STMT, &yymsp[0].minor.yy0); } +#line 5884 "sql.c" break; case 66: /* dnode_endpoint ::= NK_STRING */ case 67: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==67); @@ -5780,7 +5917,9 @@ static YYACTIONTYPE yy_reduce( case 575: /* star_func ::= FIRST */ yytestcase(yyruleno==575); case 576: /* star_func ::= LAST */ yytestcase(yyruleno==576); case 577: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==577); +#line 165 "sql.y" { yylhsminor.yy649 = yymsp[0].minor.yy0; } +#line 5922 "sql.c" yymsp[0].minor.yy649 = yylhsminor.yy649; break; case 69: /* force_opt ::= */ @@ -5792,7 +5931,9 @@ static YYACTIONTYPE yy_reduce( case 422: /* ignore_opt ::= */ yytestcase(yyruleno==422); case 657: /* tag_mode_opt ::= */ yytestcase(yyruleno==657); case 659: /* set_quantifier_opt ::= */ yytestcase(yyruleno==659); +#line 171 "sql.y" { yymsp[1].minor.yy705 = false; } +#line 5936 "sql.c" break; case 70: /* force_opt ::= FORCE */ case 71: /* unsafe_opt ::= UNSAFE */ yytestcase(yyruleno==71); @@ -5800,318 +5941,486 @@ static YYACTIONTYPE yy_reduce( case 386: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==386); case 658: /* tag_mode_opt ::= TAGS */ yytestcase(yyruleno==658); case 660: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==660); +#line 172 "sql.y" { yymsp[0].minor.yy705 = true; } +#line 5946 "sql.c" break; case 72: /* cmd ::= ALTER CLUSTER NK_STRING */ +#line 179 "sql.y" { pCxt->pRootNode = createAlterClusterStmt(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 5951 "sql.c" break; case 73: /* cmd ::= ALTER CLUSTER NK_STRING NK_STRING */ +#line 180 "sql.y" { pCxt->pRootNode = createAlterClusterStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } +#line 5956 "sql.c" break; case 74: /* cmd ::= ALTER LOCAL NK_STRING */ +#line 183 "sql.y" { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 5961 "sql.c" break; case 75: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ +#line 184 "sql.y" { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } +#line 5966 "sql.c" break; case 76: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ +#line 187 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } +#line 5971 "sql.c" break; case 77: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ +#line 188 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } +#line 5976 "sql.c" break; case 78: /* cmd ::= RESTORE QNODE ON DNODE NK_INTEGER */ +#line 189 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_QNODE_STMT, &yymsp[0].minor.yy0); } +#line 5981 "sql.c" break; case 79: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ +#line 192 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } +#line 5986 "sql.c" break; case 80: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ +#line 193 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } +#line 5991 "sql.c" break; case 81: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ +#line 196 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } +#line 5996 "sql.c" break; case 82: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ +#line 197 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } +#line 6001 "sql.c" break; case 83: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ +#line 200 "sql.y" { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } +#line 6006 "sql.c" break; case 84: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ +#line 201 "sql.y" { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } +#line 6011 "sql.c" break; case 85: /* cmd ::= RESTORE MNODE ON DNODE NK_INTEGER */ +#line 202 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_MNODE_STMT, &yymsp[0].minor.yy0); } +#line 6016 "sql.c" break; case 86: /* cmd ::= RESTORE VNODE ON DNODE NK_INTEGER */ +#line 205 "sql.y" { pCxt->pRootNode = createRestoreComponentNodeStmt(pCxt, QUERY_NODE_RESTORE_VNODE_STMT, &yymsp[0].minor.yy0); } +#line 6021 "sql.c" break; case 87: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +#line 208 "sql.y" { pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy705, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); } +#line 6026 "sql.c" break; case 88: /* cmd ::= DROP DATABASE exists_opt db_name */ +#line 209 "sql.y" { pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 6031 "sql.c" break; case 89: /* cmd ::= USE db_name */ +#line 210 "sql.y" { pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy649); } +#line 6036 "sql.c" break; case 90: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +#line 211 "sql.y" { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); } +#line 6041 "sql.c" break; case 91: /* cmd ::= FLUSH DATABASE db_name */ +#line 212 "sql.y" { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy649); } +#line 6046 "sql.c" break; case 92: /* cmd ::= TRIM DATABASE db_name speed_opt */ +#line 213 "sql.y" { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy756); } +#line 6051 "sql.c" break; case 93: /* cmd ::= S3MIGRATE DATABASE db_name */ +#line 214 "sql.y" { pCxt->pRootNode = createS3MigrateDatabaseStmt(pCxt, &yymsp[0].minor.yy649); } +#line 6056 "sql.c" break; case 94: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ +#line 215 "sql.y" { pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 6061 "sql.c" break; case 95: /* not_exists_opt ::= IF NOT EXISTS */ +#line 219 "sql.y" { yymsp[-2].minor.yy705 = true; } +#line 6066 "sql.c" break; case 97: /* exists_opt ::= IF EXISTS */ case 392: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==392); case 423: /* ignore_opt ::= IGNORE UNTREATED */ yytestcase(yyruleno==423); +#line 224 "sql.y" { yymsp[-1].minor.yy705 = true; } +#line 6073 "sql.c" break; case 99: /* db_options ::= */ +#line 227 "sql.y" { yymsp[1].minor.yy600 = createDefaultDatabaseOptions(pCxt); } +#line 6078 "sql.c" break; case 100: /* db_options ::= db_options BUFFER NK_INTEGER */ +#line 228 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } +#line 6083 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 101: /* db_options ::= db_options CACHEMODEL NK_STRING */ +#line 229 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } +#line 6089 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 102: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +#line 230 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } +#line 6095 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 103: /* db_options ::= db_options COMP NK_INTEGER */ +#line 231 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_COMP, &yymsp[0].minor.yy0); } +#line 6101 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 104: /* db_options ::= db_options DURATION NK_INTEGER */ case 105: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==105); +#line 232 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } +#line 6108 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 106: /* db_options ::= db_options MAXROWS NK_INTEGER */ +#line 234 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } +#line 6114 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 107: /* db_options ::= db_options MINROWS NK_INTEGER */ +#line 235 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } +#line 6120 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 108: /* db_options ::= db_options KEEP integer_list */ case 109: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==109); +#line 236 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP, yymsp[0].minor.yy748); } +#line 6127 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 110: /* db_options ::= db_options PAGES NK_INTEGER */ +#line 238 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } +#line 6133 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 111: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +#line 239 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } +#line 6139 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 112: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +#line 240 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } +#line 6145 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 113: /* db_options ::= db_options PRECISION NK_STRING */ +#line 241 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } +#line 6151 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 114: /* db_options ::= db_options REPLICA NK_INTEGER */ +#line 242 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } +#line 6157 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 115: /* db_options ::= db_options VGROUPS NK_INTEGER */ +#line 244 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } +#line 6163 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 116: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +#line 245 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } +#line 6169 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 117: /* db_options ::= db_options RETENTIONS retention_list */ +#line 246 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_RETENTIONS, yymsp[0].minor.yy748); } +#line 6175 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 118: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +#line 247 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } +#line 6181 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 119: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +#line 248 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL, &yymsp[0].minor.yy0); } +#line 6187 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 120: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +#line 249 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } +#line 6193 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 121: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +#line 250 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } +#line 6199 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 122: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ +#line 251 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &t); } +#line 6209 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 123: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +#line 256 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } +#line 6215 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 124: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ +#line 257 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &t); } +#line 6225 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 125: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +#line 262 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } +#line 6231 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 126: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +#line 263 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } +#line 6237 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 127: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +#line 264 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } +#line 6243 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 128: /* db_options ::= db_options TABLE_PREFIX signed */ +#line 265 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_PREFIX, yymsp[0].minor.yy600); } +#line 6249 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 129: /* db_options ::= db_options TABLE_SUFFIX signed */ +#line 266 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_SUFFIX, yymsp[0].minor.yy600); } +#line 6255 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 130: /* db_options ::= db_options S3_CHUNKSIZE NK_INTEGER */ +#line 267 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_CHUNKSIZE, &yymsp[0].minor.yy0); } +#line 6261 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 131: /* db_options ::= db_options S3_KEEPLOCAL NK_INTEGER */ case 132: /* db_options ::= db_options S3_KEEPLOCAL NK_VARIABLE */ yytestcase(yyruleno==132); +#line 268 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_KEEPLOCAL, &yymsp[0].minor.yy0); } +#line 6268 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 133: /* db_options ::= db_options S3_COMPACT NK_INTEGER */ +#line 270 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_S3_COMPACT, &yymsp[0].minor.yy0); } +#line 6274 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 134: /* db_options ::= db_options KEEP_TIME_OFFSET NK_INTEGER */ +#line 271 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP_TIME_OFFSET, &yymsp[0].minor.yy0); } +#line 6280 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 135: /* db_options ::= db_options ENCRYPT_ALGORITHM NK_STRING */ +#line 272 "sql.y" { yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_ENCRYPT_ALGORITHM, &yymsp[0].minor.yy0); } +#line 6286 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 136: /* alter_db_options ::= alter_db_option */ +#line 274 "sql.y" { yylhsminor.yy600 = createAlterDatabaseOptions(pCxt); yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yylhsminor.yy600, &yymsp[0].minor.yy145); } +#line 6292 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 137: /* alter_db_options ::= alter_db_options alter_db_option */ +#line 275 "sql.y" { yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy600, &yymsp[0].minor.yy145); } +#line 6298 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 138: /* alter_db_option ::= BUFFER NK_INTEGER */ +#line 279 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6304 "sql.c" break; case 139: /* alter_db_option ::= CACHEMODEL NK_STRING */ +#line 280 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6309 "sql.c" break; case 140: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +#line 281 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6314 "sql.c" break; case 141: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +#line 282 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6319 "sql.c" break; case 142: /* alter_db_option ::= KEEP integer_list */ case 143: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==143); +#line 283 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy748; } +#line 6325 "sql.c" break; case 144: /* alter_db_option ::= PAGES NK_INTEGER */ +#line 285 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_PAGES; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6330 "sql.c" break; case 145: /* alter_db_option ::= REPLICA NK_INTEGER */ +#line 286 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6335 "sql.c" break; case 146: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +#line 288 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_WAL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6340 "sql.c" break; case 147: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +#line 289 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6345 "sql.c" break; case 148: /* alter_db_option ::= MINROWS NK_INTEGER */ +#line 290 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6350 "sql.c" break; case 149: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ +#line 291 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6355 "sql.c" break; case 150: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ +#line 292 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yymsp[-2].minor.yy145.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy145.val = t; } +#line 6364 "sql.c" break; case 151: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ +#line 297 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6369 "sql.c" break; case 152: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ +#line 298 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yymsp[-2].minor.yy145.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy145.val = t; } +#line 6378 "sql.c" break; case 153: /* alter_db_option ::= S3_KEEPLOCAL NK_INTEGER */ case 154: /* alter_db_option ::= S3_KEEPLOCAL NK_VARIABLE */ yytestcase(yyruleno==154); +#line 303 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_S3_KEEPLOCAL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6384 "sql.c" break; case 155: /* alter_db_option ::= S3_COMPACT NK_INTEGER */ +#line 305 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_S3_COMPACT, yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6389 "sql.c" break; case 156: /* alter_db_option ::= KEEP_TIME_OFFSET NK_INTEGER */ +#line 306 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_KEEP_TIME_OFFSET; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6394 "sql.c" break; case 157: /* alter_db_option ::= ENCRYPT_ALGORITHM NK_STRING */ +#line 307 "sql.y" { yymsp[-1].minor.yy145.type = DB_OPTION_ENCRYPT_ALGORITHM; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6399 "sql.c" break; case 158: /* integer_list ::= NK_INTEGER */ +#line 311 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 6404 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 159: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ case 437: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==437); +#line 312 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 6411 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 160: /* variable_list ::= NK_VARIABLE */ +#line 316 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 6417 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 161: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +#line 317 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 6423 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 162: /* retention_list ::= retention */ @@ -6131,7 +6440,9 @@ static YYACTIONTYPE yy_reduce( case 662: /* select_list ::= select_item */ yytestcase(yyruleno==662); case 673: /* partition_list ::= partition_item */ yytestcase(yyruleno==673); case 734: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==734); +#line 321 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, yymsp[0].minor.yy600); } +#line 6445 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 163: /* retention_list ::= retention_list NK_COMMA retention */ @@ -6149,750 +6460,1178 @@ static YYACTIONTYPE yy_reduce( case 663: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==663); case 674: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==674); case 735: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==735); +#line 322 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); } +#line 6465 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 164: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ case 165: /* retention ::= NK_MINUS NK_COLON NK_VARIABLE */ yytestcase(yyruleno==165); +#line 324 "sql.y" { yylhsminor.yy600 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 6472 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 166: /* speed_opt ::= */ case 387: /* bufsize_opt ::= */ yytestcase(yyruleno==387); +#line 329 "sql.y" { yymsp[1].minor.yy756 = 0; } +#line 6479 "sql.c" break; case 167: /* speed_opt ::= BWLIMIT NK_INTEGER */ case 388: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==388); +#line 330 "sql.y" { yymsp[-1].minor.yy756 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } +#line 6485 "sql.c" break; case 169: /* start_opt ::= START WITH NK_INTEGER */ case 173: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==173); +#line 333 "sql.y" { yymsp[-2].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +#line 6491 "sql.c" break; case 170: /* start_opt ::= START WITH NK_STRING */ case 174: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==174); +#line 334 "sql.y" { yymsp[-2].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 6497 "sql.c" break; case 171: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ case 175: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==175); +#line 335 "sql.y" { yymsp[-3].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 6503 "sql.c" break; case 176: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 178: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==178); +#line 344 "sql.y" { pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy705, yymsp[-5].minor.yy600, yymsp[-3].minor.yy748, yymsp[-1].minor.yy748, yymsp[0].minor.yy600); } +#line 6509 "sql.c" break; case 177: /* cmd ::= CREATE TABLE multi_create_clause */ +#line 345 "sql.y" { pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy748); } +#line 6514 "sql.c" break; case 179: /* cmd ::= DROP TABLE multi_drop_clause */ +#line 348 "sql.y" { pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy748); } +#line 6519 "sql.c" break; case 180: /* cmd ::= DROP STABLE exists_opt full_table_name */ +#line 349 "sql.y" { pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); } +#line 6524 "sql.c" break; case 181: /* cmd ::= ALTER TABLE alter_table_clause */ case 439: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==439); case 440: /* cmd ::= insert_query */ yytestcase(yyruleno==440); +#line 351 "sql.y" { pCxt->pRootNode = yymsp[0].minor.yy600; } +#line 6531 "sql.c" break; case 182: /* cmd ::= ALTER STABLE alter_table_clause */ +#line 352 "sql.y" { pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy600); } +#line 6536 "sql.c" break; case 183: /* alter_table_clause ::= full_table_name alter_table_options */ +#line 354 "sql.y" { yylhsminor.yy600 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 6541 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; - case 184: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + case 184: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name column_options */ +#line 356 "sql.y" +{ yylhsminor.yy600 = createAlterTableAddModifyColOptions2(pCxt, yymsp[-5].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy400, yymsp[0].minor.yy600); } +#line 6547 "sql.c" + yymsp[-5].minor.yy600 = yylhsminor.yy600; break; case 185: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +#line 357 "sql.y" { yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy649); } +#line 6553 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 186: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +#line 359 "sql.y" { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); } +#line 6559 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 187: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name column_options */ +#line 361 "sql.y" { yylhsminor.yy600 = createAlterTableAddModifyColOptions(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS, &yymsp[-1].minor.yy649, yymsp[0].minor.yy600); } +#line 6565 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 188: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +#line 363 "sql.y" { yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); } +#line 6571 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 189: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +#line 365 "sql.y" { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); } +#line 6577 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 190: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +#line 366 "sql.y" { yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy649); } +#line 6583 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 191: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +#line 368 "sql.y" { yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400); } +#line 6589 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 192: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +#line 370 "sql.y" { yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); } +#line 6595 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 193: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ tags_literal */ +#line 372 "sql.y" { yylhsminor.yy600 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy600, &yymsp[-2].minor.yy649, yymsp[0].minor.yy600); } +#line 6601 "sql.c" yymsp[-5].minor.yy600 = yylhsminor.yy600; break; case 195: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 587: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==587); +#line 377 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-1].minor.yy748, yymsp[0].minor.yy600); } +#line 6608 "sql.c" yymsp[-1].minor.yy748 = yylhsminor.yy748; break; case 196: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP tags_literal_list NK_RP table_options */ +#line 381 "sql.y" { yylhsminor.yy600 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy705, yymsp[-8].minor.yy600, yymsp[-6].minor.yy600, yymsp[-5].minor.yy748, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); } +#line 6614 "sql.c" yymsp[-9].minor.yy600 = yylhsminor.yy600; break; case 199: /* drop_table_clause ::= exists_opt full_table_name */ +#line 388 "sql.y" { yylhsminor.yy600 = createDropTableClause(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); } +#line 6620 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 201: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ case 402: /* col_list_opt ::= NK_LP column_stream_def_list NK_RP */ yytestcase(yyruleno==402); +#line 393 "sql.y" { yymsp[-2].minor.yy748 = yymsp[-1].minor.yy748; } +#line 6627 "sql.c" break; case 202: /* full_table_name ::= table_name */ case 343: /* full_tsma_name ::= tsma_name */ yytestcase(yyruleno==343); +#line 395 "sql.y" { yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy649, NULL); } +#line 6633 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 203: /* full_table_name ::= db_name NK_DOT table_name */ case 344: /* full_tsma_name ::= db_name NK_DOT tsma_name */ yytestcase(yyruleno==344); +#line 396 "sql.y" { yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649, NULL); } +#line 6640 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 206: /* tag_def ::= column_name type_name */ +#line 402 "sql.y" { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy649, yymsp[0].minor.yy400, NULL); } +#line 6646 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 209: /* column_def ::= column_name type_name column_options */ +#line 410 "sql.y" { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-2].minor.yy649, yymsp[-1].minor.yy400, yymsp[0].minor.yy600); } +#line 6652 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 210: /* type_name ::= BOOL */ +#line 414 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BOOL); } +#line 6658 "sql.c" break; case 211: /* type_name ::= TINYINT */ +#line 415 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TINYINT); } +#line 6663 "sql.c" break; case 212: /* type_name ::= SMALLINT */ +#line 416 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +#line 6668 "sql.c" break; case 213: /* type_name ::= INT */ case 214: /* type_name ::= INTEGER */ yytestcase(yyruleno==214); +#line 417 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_INT); } +#line 6674 "sql.c" break; case 215: /* type_name ::= BIGINT */ +#line 419 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BIGINT); } +#line 6679 "sql.c" break; case 216: /* type_name ::= FLOAT */ +#line 420 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_FLOAT); } +#line 6684 "sql.c" break; case 217: /* type_name ::= DOUBLE */ +#line 421 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +#line 6689 "sql.c" break; case 218: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +#line 422 "sql.y" { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +#line 6694 "sql.c" break; case 219: /* type_name ::= TIMESTAMP */ +#line 423 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +#line 6699 "sql.c" break; case 220: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +#line 424 "sql.y" { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +#line 6704 "sql.c" break; case 221: /* type_name ::= TINYINT UNSIGNED */ +#line 425 "sql.y" { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +#line 6709 "sql.c" break; case 222: /* type_name ::= SMALLINT UNSIGNED */ +#line 426 "sql.y" { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +#line 6714 "sql.c" break; case 223: /* type_name ::= INT UNSIGNED */ +#line 427 "sql.y" { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UINT); } +#line 6719 "sql.c" break; case 224: /* type_name ::= BIGINT UNSIGNED */ +#line 428 "sql.y" { yymsp[-1].minor.yy400 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +#line 6724 "sql.c" break; case 225: /* type_name ::= JSON */ +#line 429 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_JSON); } +#line 6729 "sql.c" break; case 226: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +#line 430 "sql.y" { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +#line 6734 "sql.c" break; case 227: /* type_name ::= MEDIUMBLOB */ +#line 431 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +#line 6739 "sql.c" break; case 228: /* type_name ::= BLOB */ +#line 432 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_BLOB); } +#line 6744 "sql.c" break; case 229: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +#line 433 "sql.y" { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +#line 6749 "sql.c" break; case 230: /* type_name ::= GEOMETRY NK_LP NK_INTEGER NK_RP */ +#line 434 "sql.y" { yymsp[-3].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_GEOMETRY, &yymsp[-1].minor.yy0); } +#line 6754 "sql.c" break; case 231: /* type_name ::= DECIMAL */ +#line 435 "sql.y" { yymsp[0].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6759 "sql.c" break; case 232: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +#line 436 "sql.y" { yymsp[-3].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6764 "sql.c" break; case 233: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +#line 437 "sql.y" { yymsp[-5].minor.yy400 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +#line 6769 "sql.c" break; case 234: /* type_name_default_len ::= BINARY */ +#line 441 "sql.y" { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, NULL); } +#line 6774 "sql.c" break; case 235: /* type_name_default_len ::= NCHAR */ +#line 442 "sql.y" { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, NULL); } +#line 6779 "sql.c" break; case 236: /* type_name_default_len ::= VARCHAR */ +#line 443 "sql.y" { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, NULL); } +#line 6784 "sql.c" break; case 237: /* type_name_default_len ::= VARBINARY */ +#line 444 "sql.y" { yymsp[0].minor.yy400 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, NULL); } +#line 6789 "sql.c" break; case 240: /* tags_def ::= TAGS NK_LP tag_def_list NK_RP */ case 410: /* tag_def_or_ref_opt ::= TAGS NK_LP column_stream_def_list NK_RP */ yytestcase(yyruleno==410); +#line 453 "sql.y" { yymsp[-3].minor.yy748 = yymsp[-1].minor.yy748; } +#line 6795 "sql.c" break; case 241: /* table_options ::= */ +#line 455 "sql.y" { yymsp[1].minor.yy600 = createDefaultTableOptions(pCxt); } +#line 6800 "sql.c" break; case 242: /* table_options ::= table_options COMMENT NK_STRING */ +#line 456 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } +#line 6805 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 243: /* table_options ::= table_options MAX_DELAY duration_list */ +#line 457 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy748); } +#line 6811 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 244: /* table_options ::= table_options WATERMARK duration_list */ +#line 458 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy748); } +#line 6817 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 245: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +#line 459 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy748); } +#line 6823 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 246: /* table_options ::= table_options TTL NK_INTEGER */ +#line 460 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } +#line 6829 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 247: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +#line 461 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_SMA, yymsp[-1].minor.yy748); } +#line 6835 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 248: /* table_options ::= table_options DELETE_MARK duration_list */ +#line 462 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy748); } +#line 6841 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 249: /* alter_table_options ::= alter_table_option */ +#line 464 "sql.y" { yylhsminor.yy600 = createAlterTableOptions(pCxt); yylhsminor.yy600 = setTableOption(pCxt, yylhsminor.yy600, yymsp[0].minor.yy145.type, &yymsp[0].minor.yy145.val); } +#line 6847 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 250: /* alter_table_options ::= alter_table_options alter_table_option */ +#line 465 "sql.y" { yylhsminor.yy600 = setTableOption(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy145.type, &yymsp[0].minor.yy145.val); } +#line 6853 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 251: /* alter_table_option ::= COMMENT NK_STRING */ +#line 469 "sql.y" { yymsp[-1].minor.yy145.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6859 "sql.c" break; case 252: /* alter_table_option ::= TTL NK_INTEGER */ +#line 470 "sql.y" { yymsp[-1].minor.yy145.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy145.val = yymsp[0].minor.yy0; } +#line 6864 "sql.c" break; case 253: /* duration_list ::= duration_literal */ case 539: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==539); +#line 474 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 6870 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 254: /* duration_list ::= duration_list NK_COMMA duration_literal */ case 540: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==540); +#line 475 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 6877 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 257: /* rollup_func_name ::= function_name */ +#line 482 "sql.y" { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy649, NULL); } +#line 6883 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 258: /* rollup_func_name ::= FIRST */ case 259: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==259); case 332: /* tag_item ::= QTAGS */ yytestcase(yyruleno==332); +#line 483 "sql.y" { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 6891 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 262: /* col_name ::= column_name */ case 333: /* tag_item ::= column_name */ yytestcase(yyruleno==333); +#line 491 "sql.y" { yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy649); } +#line 6898 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 263: /* cmd ::= SHOW DNODES */ +#line 494 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } +#line 6904 "sql.c" break; case 264: /* cmd ::= SHOW USERS */ +#line 495 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } +#line 6909 "sql.c" break; case 265: /* cmd ::= SHOW USER PRIVILEGES */ +#line 496 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } +#line 6914 "sql.c" break; case 266: /* cmd ::= SHOW db_kind_opt DATABASES */ +#line 497 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); setShowKind(pCxt, pCxt->pRootNode, yymsp[-1].minor.yy245); } +#line 6922 "sql.c" break; case 267: /* cmd ::= SHOW table_kind_db_name_cond_opt TABLES like_pattern_opt */ +#line 501 "sql.y" { pCxt->pRootNode = createShowTablesStmt(pCxt, yymsp[-2].minor.yy125, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +#line 6929 "sql.c" break; case 268: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +#line 504 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +#line 6934 "sql.c" break; case 269: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +#line 505 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy600, NULL, OP_TYPE_LIKE); } +#line 6939 "sql.c" break; case 270: /* cmd ::= SHOW MNODES */ +#line 506 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } +#line 6944 "sql.c" break; case 271: /* cmd ::= SHOW QNODES */ +#line 508 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } +#line 6949 "sql.c" break; case 272: /* cmd ::= SHOW ARBGROUPS */ +#line 509 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ARBGROUPS_STMT); } +#line 6954 "sql.c" break; case 273: /* cmd ::= SHOW FUNCTIONS */ +#line 510 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } +#line 6959 "sql.c" break; case 274: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +#line 511 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +#line 6964 "sql.c" break; case 275: /* cmd ::= SHOW INDEXES FROM db_name NK_DOT table_name */ +#line 512 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), OP_TYPE_EQUAL); } +#line 6969 "sql.c" break; case 276: /* cmd ::= SHOW STREAMS */ +#line 513 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } +#line 6974 "sql.c" break; case 277: /* cmd ::= SHOW ACCOUNTS */ +#line 514 "sql.y" { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } +#line 6979 "sql.c" break; case 278: /* cmd ::= SHOW APPS */ +#line 515 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } +#line 6984 "sql.c" break; case 279: /* cmd ::= SHOW CONNECTIONS */ +#line 516 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } +#line 6989 "sql.c" break; case 280: /* cmd ::= SHOW LICENCES */ case 281: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==281); +#line 517 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } +#line 6995 "sql.c" break; case 282: /* cmd ::= SHOW GRANTS FULL */ +#line 519 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_FULL_STMT); } +#line 7000 "sql.c" break; case 283: /* cmd ::= SHOW GRANTS LOGS */ +#line 520 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_LOGS_STMT); } +#line 7005 "sql.c" break; case 284: /* cmd ::= SHOW CLUSTER MACHINES */ +#line 521 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); } +#line 7010 "sql.c" break; case 285: /* cmd ::= SHOW CREATE DATABASE db_name */ +#line 522 "sql.y" { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy649); } +#line 7015 "sql.c" break; case 286: /* cmd ::= SHOW CREATE TABLE full_table_name */ +#line 523 "sql.y" { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy600); } +#line 7020 "sql.c" break; case 287: /* cmd ::= SHOW CREATE STABLE full_table_name */ +#line 524 "sql.y" { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy600); } +#line 7026 "sql.c" break; case 288: /* cmd ::= SHOW ENCRYPTIONS */ +#line 526 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_ENCRYPTIONS_STMT); } +#line 7031 "sql.c" break; case 289: /* cmd ::= SHOW QUERIES */ +#line 527 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } +#line 7036 "sql.c" break; case 290: /* cmd ::= SHOW SCORES */ +#line 528 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } +#line 7041 "sql.c" break; case 291: /* cmd ::= SHOW TOPICS */ +#line 529 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } +#line 7046 "sql.c" break; case 292: /* cmd ::= SHOW VARIABLES */ case 293: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==293); +#line 530 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } +#line 7052 "sql.c" break; case 294: /* cmd ::= SHOW LOCAL VARIABLES */ +#line 532 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } +#line 7057 "sql.c" break; case 295: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +#line 533 "sql.y" { pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy600); } +#line 7062 "sql.c" break; case 296: /* cmd ::= SHOW BNODES */ +#line 534 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } +#line 7067 "sql.c" break; case 297: /* cmd ::= SHOW SNODES */ +#line 535 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } +#line 7072 "sql.c" break; case 298: /* cmd ::= SHOW CLUSTER */ +#line 536 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } +#line 7077 "sql.c" break; case 299: /* cmd ::= SHOW TRANSACTIONS */ +#line 537 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } +#line 7082 "sql.c" break; case 300: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +#line 538 "sql.y" { pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy600); } +#line 7087 "sql.c" break; case 301: /* cmd ::= SHOW CONSUMERS */ +#line 539 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } +#line 7092 "sql.c" break; case 302: /* cmd ::= SHOW SUBSCRIPTIONS */ +#line 540 "sql.y" { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } +#line 7097 "sql.c" break; case 303: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +#line 541 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } +#line 7102 "sql.c" break; case 304: /* cmd ::= SHOW TAGS FROM db_name NK_DOT table_name */ +#line 542 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), OP_TYPE_EQUAL); } +#line 7107 "sql.c" break; case 305: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +#line 543 "sql.y" { pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600, yymsp[-3].minor.yy748); } +#line 7112 "sql.c" break; case 306: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM db_name NK_DOT table_name */ +#line 544 "sql.y" { pCxt->pRootNode = createShowTableTagsStmt(pCxt, createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649), createIdentifierValueNode(pCxt, &yymsp[-2].minor.yy649), yymsp[-4].minor.yy748); } +#line 7117 "sql.c" break; case 307: /* cmd ::= SHOW VNODES ON DNODE NK_INTEGER */ +#line 545 "sql.y" { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } +#line 7122 "sql.c" break; case 308: /* cmd ::= SHOW VNODES */ +#line 546 "sql.y" { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, NULL); } +#line 7127 "sql.c" break; case 309: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +#line 548 "sql.y" { pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy600, QUERY_NODE_SHOW_DB_ALIVE_STMT); } +#line 7132 "sql.c" break; case 310: /* cmd ::= SHOW CLUSTER ALIVE */ +#line 549 "sql.y" { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } +#line 7137 "sql.c" break; case 311: /* cmd ::= SHOW db_name_cond_opt VIEWS like_pattern_opt */ +#line 550 "sql.y" { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } +#line 7142 "sql.c" break; case 312: /* cmd ::= SHOW CREATE VIEW full_table_name */ +#line 551 "sql.y" { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, yymsp[0].minor.yy600); } +#line 7147 "sql.c" break; case 313: /* cmd ::= SHOW COMPACTS */ +#line 552 "sql.y" { pCxt->pRootNode = createShowCompactsStmt(pCxt, QUERY_NODE_SHOW_COMPACTS_STMT); } +#line 7152 "sql.c" break; case 314: /* cmd ::= SHOW COMPACT NK_INTEGER */ +#line 553 "sql.y" { pCxt->pRootNode = createShowCompactDetailsStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 7157 "sql.c" break; case 315: /* table_kind_db_name_cond_opt ::= */ +#line 557 "sql.y" { yymsp[1].minor.yy125.kind = SHOW_KIND_ALL; yymsp[1].minor.yy125.dbName = nil_token; } +#line 7162 "sql.c" break; case 316: /* table_kind_db_name_cond_opt ::= table_kind */ +#line 558 "sql.y" { yylhsminor.yy125.kind = yymsp[0].minor.yy245; yylhsminor.yy125.dbName = nil_token; } +#line 7167 "sql.c" yymsp[0].minor.yy125 = yylhsminor.yy125; break; case 317: /* table_kind_db_name_cond_opt ::= db_name NK_DOT */ +#line 559 "sql.y" { yylhsminor.yy125.kind = SHOW_KIND_ALL; yylhsminor.yy125.dbName = yymsp[-1].minor.yy649; } +#line 7173 "sql.c" yymsp[-1].minor.yy125 = yylhsminor.yy125; break; case 318: /* table_kind_db_name_cond_opt ::= table_kind db_name NK_DOT */ +#line 560 "sql.y" { yylhsminor.yy125.kind = yymsp[-2].minor.yy245; yylhsminor.yy125.dbName = yymsp[-1].minor.yy649; } +#line 7179 "sql.c" yymsp[-2].minor.yy125 = yylhsminor.yy125; break; case 319: /* table_kind ::= NORMAL */ +#line 564 "sql.y" { yymsp[0].minor.yy245 = SHOW_KIND_TABLES_NORMAL; } +#line 7185 "sql.c" break; case 320: /* table_kind ::= CHILD */ +#line 565 "sql.y" { yymsp[0].minor.yy245 = SHOW_KIND_TABLES_CHILD; } +#line 7190 "sql.c" break; case 321: /* db_name_cond_opt ::= */ case 326: /* from_db_opt ::= */ yytestcase(yyruleno==326); +#line 567 "sql.y" { yymsp[1].minor.yy600 = createDefaultDatabaseCondValue(pCxt); } +#line 7196 "sql.c" break; case 322: /* db_name_cond_opt ::= db_name NK_DOT */ +#line 568 "sql.y" { yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy649); } +#line 7201 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 324: /* like_pattern_opt ::= LIKE NK_STRING */ +#line 571 "sql.y" { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +#line 7207 "sql.c" break; case 325: /* table_name_cond ::= table_name */ +#line 573 "sql.y" { yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649); } +#line 7212 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 327: /* from_db_opt ::= FROM db_name */ +#line 576 "sql.y" { yymsp[-1].minor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy649); } +#line 7218 "sql.c" break; case 331: /* tag_item ::= TBNAME */ +#line 584 "sql.y" { yylhsminor.yy600 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } +#line 7223 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 334: /* tag_item ::= column_name column_alias */ +#line 587 "sql.y" { yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy649), &yymsp[0].minor.yy649); } +#line 7229 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 335: /* tag_item ::= column_name AS column_alias */ +#line 588 "sql.y" { yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy649), &yymsp[0].minor.yy649); } +#line 7235 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 336: /* db_kind_opt ::= */ +#line 592 "sql.y" { yymsp[1].minor.yy245 = SHOW_KIND_ALL; } +#line 7241 "sql.c" break; case 337: /* db_kind_opt ::= USER */ +#line 593 "sql.y" { yymsp[0].minor.yy245 = SHOW_KIND_DATABASES_USER; } +#line 7246 "sql.c" break; case 338: /* db_kind_opt ::= SYSTEM */ +#line 594 "sql.y" { yymsp[0].minor.yy245 = SHOW_KIND_DATABASES_SYSTEM; } +#line 7251 "sql.c" break; case 339: /* cmd ::= CREATE TSMA not_exists_opt tsma_name ON full_table_name tsma_func_list INTERVAL NK_LP duration_literal NK_RP */ +#line 600 "sql.y" { pCxt->pRootNode = createCreateTSMAStmt(pCxt, yymsp[-8].minor.yy705, &yymsp[-7].minor.yy649, yymsp[-4].minor.yy600, yymsp[-5].minor.yy600, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 7256 "sql.c" break; case 340: /* cmd ::= CREATE RECURSIVE TSMA not_exists_opt tsma_name ON full_table_name INTERVAL NK_LP duration_literal NK_RP */ +#line 602 "sql.y" { pCxt->pRootNode = createCreateTSMAStmt(pCxt, yymsp[-7].minor.yy705, &yymsp[-6].minor.yy649, NULL, yymsp[-4].minor.yy600, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 7261 "sql.c" break; case 341: /* cmd ::= DROP TSMA exists_opt full_tsma_name */ +#line 603 "sql.y" { pCxt->pRootNode = createDropTSMAStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); } +#line 7266 "sql.c" break; case 342: /* cmd ::= SHOW db_name_cond_opt TSMAS */ +#line 604 "sql.y" { pCxt->pRootNode = createShowTSMASStmt(pCxt, yymsp[-1].minor.yy600); } +#line 7271 "sql.c" break; case 345: /* tsma_func_list ::= FUNCTION NK_LP func_list NK_RP */ +#line 611 "sql.y" { yymsp[-3].minor.yy600 = createTSMAOptions(pCxt, yymsp[-1].minor.yy748); } +#line 7276 "sql.c" break; case 346: /* cmd ::= CREATE SMA INDEX not_exists_opt col_name ON full_table_name index_options */ +#line 615 "sql.y" { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy705, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } +#line 7281 "sql.c" break; case 347: /* cmd ::= CREATE INDEX not_exists_opt col_name ON full_table_name NK_LP col_name_list NK_RP */ +#line 617 "sql.y" { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy705, yymsp[-5].minor.yy600, yymsp[-3].minor.yy600, yymsp[-1].minor.yy748, NULL); } +#line 7286 "sql.c" break; case 348: /* cmd ::= DROP INDEX exists_opt full_index_name */ +#line 618 "sql.y" { pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); } +#line 7291 "sql.c" break; case 349: /* full_index_name ::= index_name */ +#line 620 "sql.y" { yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy649); } +#line 7296 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 350: /* full_index_name ::= db_name NK_DOT index_name */ +#line 621 "sql.y" { yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); } +#line 7302 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 351: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +#line 624 "sql.y" { yymsp[-9].minor.yy600 = createIndexOption(pCxt, yymsp[-7].minor.yy748, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 7308 "sql.c" break; case 352: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +#line 627 "sql.y" { yymsp[-11].minor.yy600 = createIndexOption(pCxt, yymsp[-9].minor.yy748, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 7313 "sql.c" break; case 355: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +#line 634 "sql.y" { yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[-3].minor.yy649, yymsp[-1].minor.yy748); } +#line 7318 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 356: /* sma_func_name ::= function_name */ case 630: /* alias_opt ::= table_alias */ yytestcase(yyruleno==630); +#line 638 "sql.y" { yylhsminor.yy649 = yymsp[0].minor.yy649; } +#line 7325 "sql.c" yymsp[0].minor.yy649 = yylhsminor.yy649; break; case 361: /* sma_stream_opt ::= */ case 411: /* stream_options ::= */ yytestcase(yyruleno==411); +#line 644 "sql.y" { yymsp[1].minor.yy600 = createStreamOptions(pCxt); } +#line 7332 "sql.c" break; case 362: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ +#line 645 "sql.y" { ((SStreamOptions*)yymsp[-2].minor.yy600)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } +#line 7337 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 363: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +#line 646 "sql.y" { ((SStreamOptions*)yymsp[-2].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } +#line 7343 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 364: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +#line 647 "sql.y" { ((SStreamOptions*)yymsp[-2].minor.yy600)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } +#line 7349 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 365: /* with_meta ::= AS */ +#line 652 "sql.y" { yymsp[0].minor.yy756 = 0; } +#line 7355 "sql.c" break; case 366: /* with_meta ::= WITH META AS */ +#line 653 "sql.y" { yymsp[-2].minor.yy756 = 1; } +#line 7360 "sql.c" break; case 367: /* with_meta ::= ONLY META AS */ +#line 654 "sql.y" { yymsp[-2].minor.yy756 = 2; } +#line 7365 "sql.c" break; case 368: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +#line 656 "sql.y" { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy705, &yymsp[-2].minor.yy649, yymsp[0].minor.yy600); } +#line 7370 "sql.c" break; case 369: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta DATABASE db_name */ +#line 658 "sql.y" { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy705, &yymsp[-3].minor.yy649, &yymsp[0].minor.yy649, yymsp[-2].minor.yy756); } +#line 7375 "sql.c" break; case 370: /* cmd ::= CREATE TOPIC not_exists_opt topic_name with_meta STABLE full_table_name where_clause_opt */ +#line 660 "sql.y" { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-5].minor.yy705, &yymsp[-4].minor.yy649, yymsp[-1].minor.yy600, yymsp[-3].minor.yy756, yymsp[0].minor.yy600); } +#line 7380 "sql.c" break; case 371: /* cmd ::= DROP TOPIC exists_opt topic_name */ +#line 662 "sql.y" { pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 7385 "sql.c" break; case 372: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +#line 663 "sql.y" { pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy705, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); } +#line 7390 "sql.c" break; case 373: /* cmd ::= DESC full_table_name */ case 374: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==374); +#line 666 "sql.y" { pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy600); } +#line 7396 "sql.c" break; case 375: /* cmd ::= RESET QUERY CACHE */ +#line 670 "sql.y" { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } +#line 7401 "sql.c" break; case 376: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ case 377: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==377); +#line 673 "sql.y" { pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy705, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 7407 "sql.c" break; case 380: /* explain_options ::= */ +#line 681 "sql.y" { yymsp[1].minor.yy600 = createDefaultExplainOptions(pCxt); } +#line 7412 "sql.c" break; case 381: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +#line 682 "sql.y" { yylhsminor.yy600 = setExplainVerbose(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } +#line 7417 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 382: /* explain_options ::= explain_options RATIO NK_FLOAT */ +#line 683 "sql.y" { yylhsminor.yy600 = setExplainRatio(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } +#line 7423 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 383: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ +#line 688 "sql.y" { pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy705, yymsp[-9].minor.yy705, &yymsp[-6].minor.yy649, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy400, yymsp[-1].minor.yy756, &yymsp[0].minor.yy649, yymsp[-10].minor.yy705); } +#line 7429 "sql.c" break; case 384: /* cmd ::= DROP FUNCTION exists_opt function_name */ +#line 689 "sql.y" { pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 7434 "sql.c" break; case 389: /* language_opt ::= */ case 434: /* on_vgroup_id ::= */ yytestcase(yyruleno==434); +#line 703 "sql.y" { yymsp[1].minor.yy649 = nil_token; } +#line 7440 "sql.c" break; case 390: /* language_opt ::= LANGUAGE NK_STRING */ case 435: /* on_vgroup_id ::= ON NK_INTEGER */ yytestcase(yyruleno==435); +#line 704 "sql.y" { yymsp[-1].minor.yy649 = yymsp[0].minor.yy0; } +#line 7446 "sql.c" break; case 393: /* cmd ::= CREATE or_replace_opt VIEW full_view_name AS query_or_subquery */ +#line 713 "sql.y" { pCxt->pRootNode = createCreateViewStmt(pCxt, yymsp[-4].minor.yy705, yymsp[-2].minor.yy600, &yymsp[-1].minor.yy0, yymsp[0].minor.yy600); } +#line 7451 "sql.c" break; case 394: /* cmd ::= DROP VIEW exists_opt full_view_name */ +#line 714 "sql.y" { pCxt->pRootNode = createDropViewStmt(pCxt, yymsp[-1].minor.yy705, yymsp[0].minor.yy600); } +#line 7456 "sql.c" break; case 395: /* full_view_name ::= view_name */ +#line 716 "sql.y" { yylhsminor.yy600 = createViewNode(pCxt, NULL, &yymsp[0].minor.yy649); } +#line 7461 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 396: /* full_view_name ::= db_name NK_DOT view_name */ +#line 717 "sql.y" { yylhsminor.yy600 = createViewNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649); } +#line 7467 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 397: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +#line 722 "sql.y" { pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy705, &yymsp[-8].minor.yy649, yymsp[-5].minor.yy600, yymsp[-7].minor.yy600, yymsp[-3].minor.yy748, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, yymsp[-4].minor.yy748); } +#line 7473 "sql.c" break; case 398: /* cmd ::= DROP STREAM exists_opt stream_name */ +#line 723 "sql.y" { pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 7478 "sql.c" break; case 399: /* cmd ::= PAUSE STREAM exists_opt stream_name */ +#line 724 "sql.y" { pCxt->pRootNode = createPauseStreamStmt(pCxt, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 7483 "sql.c" break; case 400: /* cmd ::= RESUME STREAM exists_opt ignore_opt stream_name */ +#line 725 "sql.y" { pCxt->pRootNode = createResumeStreamStmt(pCxt, yymsp[-2].minor.yy705, yymsp[-1].minor.yy705, &yymsp[0].minor.yy649); } +#line 7488 "sql.c" break; case 405: /* column_stream_def ::= column_name stream_col_options */ +#line 738 "sql.y" { yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy649, createDataType(TSDB_DATA_TYPE_NULL), yymsp[0].minor.yy600); } +#line 7493 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 406: /* stream_col_options ::= */ case 743: /* column_options ::= */ yytestcase(yyruleno==743); +#line 739 "sql.y" { yymsp[1].minor.yy600 = createDefaultColumnOptions(pCxt); } +#line 7500 "sql.c" break; case 407: /* stream_col_options ::= stream_col_options PRIMARY KEY */ case 744: /* column_options ::= column_options PRIMARY KEY */ yytestcase(yyruleno==744); +#line 740 "sql.y" { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_PRIMARYKEY, NULL); } +#line 7506 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 412: /* stream_options ::= stream_options TRIGGER AT_ONCE */ case 413: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==413); +#line 750 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } +#line 7513 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 414: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +#line 752 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 7519 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 415: /* stream_options ::= stream_options WATERMARK duration_literal */ +#line 753 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 7525 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 416: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +#line 754 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } +#line 7531 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 417: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +#line 755 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } +#line 7537 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 418: /* stream_options ::= stream_options DELETE_MARK duration_literal */ +#line 756 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-2].minor.yy600, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 7543 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 419: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +#line 757 "sql.y" { yylhsminor.yy600 = setStreamOptions(pCxt, yymsp[-3].minor.yy600, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } +#line 7549 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 421: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ case 687: /* sliding_opt ::= SLIDING NK_LP interval_sliding_duration_literal NK_RP */ yytestcase(yyruleno==687); case 711: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==711); +#line 760 "sql.y" { yymsp[-3].minor.yy600 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy600); } +#line 7557 "sql.c" break; case 424: /* cmd ::= KILL CONNECTION NK_INTEGER */ +#line 768 "sql.y" { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } +#line 7562 "sql.c" break; case 425: /* cmd ::= KILL QUERY NK_STRING */ +#line 769 "sql.y" { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } +#line 7567 "sql.c" break; case 426: /* cmd ::= KILL TRANSACTION NK_INTEGER */ +#line 770 "sql.y" { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } +#line 7572 "sql.c" break; case 427: /* cmd ::= KILL COMPACT NK_INTEGER */ +#line 771 "sql.y" { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_COMPACT_STMT, &yymsp[0].minor.yy0); } +#line 7577 "sql.c" break; case 428: /* cmd ::= BALANCE VGROUP */ +#line 774 "sql.y" { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } +#line 7582 "sql.c" break; case 429: /* cmd ::= BALANCE VGROUP LEADER on_vgroup_id */ +#line 775 "sql.y" { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &yymsp[0].minor.yy649); } +#line 7587 "sql.c" break; case 430: /* cmd ::= BALANCE VGROUP LEADER DATABASE db_name */ +#line 776 "sql.y" { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &yymsp[0].minor.yy649); } +#line 7592 "sql.c" break; case 431: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ +#line 777 "sql.y" { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } +#line 7597 "sql.c" break; case 432: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +#line 778 "sql.y" { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy748); } +#line 7602 "sql.c" break; case 433: /* cmd ::= SPLIT VGROUP NK_INTEGER */ +#line 779 "sql.y" { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } +#line 7607 "sql.c" break; case 436: /* dnode_list ::= DNODE NK_INTEGER */ +#line 788 "sql.y" { yymsp[-1].minor.yy748 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +#line 7612 "sql.c" break; case 438: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +#line 795 "sql.y" { pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 7617 "sql.c" break; case 441: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +#line 804 "sql.y" { yymsp[-6].minor.yy600 = createInsertStmt(pCxt, yymsp[-4].minor.yy600, yymsp[-2].minor.yy748, yymsp[0].minor.yy600); } +#line 7622 "sql.c" break; case 442: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +#line 805 "sql.y" { yymsp[-3].minor.yy600 = createInsertStmt(pCxt, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } +#line 7627 "sql.c" break; case 443: /* tags_literal ::= NK_INTEGER */ case 455: /* tags_literal ::= NK_BIN */ yytestcase(yyruleno==455); case 464: /* tags_literal ::= NK_HEX */ yytestcase(yyruleno==464); +#line 808 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0, NULL); } +#line 7634 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 444: /* tags_literal ::= NK_INTEGER NK_PLUS duration_literal */ @@ -6903,12 +7642,14 @@ yymsp[0].minor.yy600); } case 466: /* tags_literal ::= NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==466); case 474: /* tags_literal ::= NK_STRING NK_PLUS duration_literal */ yytestcase(yyruleno==474); case 475: /* tags_literal ::= NK_STRING NK_MINUS duration_literal */ yytestcase(yyruleno==475); +#line 809 "sql.y" { SToken l = yymsp[-2].minor.yy0; SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); l.n = (r.z + r.n) - l.z; yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, yymsp[0].minor.yy600); } +#line 7652 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 446: /* tags_literal ::= NK_PLUS NK_INTEGER */ @@ -6917,11 +7658,13 @@ yymsp[0].minor.yy600); } case 461: /* tags_literal ::= NK_MINUS NK_BIN */ yytestcase(yyruleno==461); case 467: /* tags_literal ::= NK_PLUS NK_HEX */ yytestcase(yyruleno==467); case 470: /* tags_literal ::= NK_MINUS NK_HEX */ yytestcase(yyruleno==470); +#line 821 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &t, NULL); } +#line 7667 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 447: /* tags_literal ::= NK_PLUS NK_INTEGER NK_PLUS duration_literal */ @@ -6936,71 +7679,97 @@ yymsp[0].minor.yy600); } case 469: /* tags_literal ::= NK_PLUS NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==469); case 471: /* tags_literal ::= NK_MINUS NK_HEX NK_PLUS duration_literal */ yytestcase(yyruleno==471); case 472: /* tags_literal ::= NK_MINUS NK_HEX NK_MINUS duration_literal */ yytestcase(yyruleno==472); +#line 826 "sql.y" { SToken l = yymsp[-3].minor.yy0; SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); l.n = (r.z + r.n) - l.z; yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, NULL, yymsp[0].minor.yy600); } +#line 7689 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 452: /* tags_literal ::= NK_FLOAT */ +#line 855 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0, NULL); } +#line 7695 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 453: /* tags_literal ::= NK_PLUS NK_FLOAT */ case 454: /* tags_literal ::= NK_MINUS NK_FLOAT */ yytestcase(yyruleno==454); +#line 856 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t, NULL); } +#line 7706 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 473: /* tags_literal ::= NK_STRING */ +#line 962 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0, NULL); } +#line 7712 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 476: /* tags_literal ::= NK_BOOL */ +#line 975 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0, NULL); } +#line 7718 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 477: /* tags_literal ::= NULL */ +#line 976 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0, NULL); } +#line 7724 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 478: /* tags_literal ::= literal_func */ +#line 978 "sql.y" { yylhsminor.yy600 = createRawValueNode(pCxt, TSDB_DATA_TYPE_BINARY, NULL, yymsp[0].minor.yy600); } +#line 7730 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 479: /* tags_literal ::= literal_func NK_PLUS duration_literal */ case 480: /* tags_literal ::= literal_func NK_MINUS duration_literal */ yytestcase(yyruleno==480); +#line 979 "sql.y" { SToken l = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken r = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); l.n = (r.z + r.n) - l.z; yylhsminor.yy600 = createRawValueNodeExt(pCxt, TSDB_DATA_TYPE_BINARY, &l, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } +#line 7742 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 483: /* literal ::= NK_INTEGER */ +#line 998 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } +#line 7748 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 484: /* literal ::= NK_FLOAT */ +#line 999 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } +#line 7754 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 485: /* literal ::= NK_STRING */ +#line 1000 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } +#line 7760 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 486: /* literal ::= NK_BOOL */ +#line 1001 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } +#line 7766 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 487: /* literal ::= TIMESTAMP NK_STRING */ +#line 1002 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } +#line 7772 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 488: /* literal ::= duration_literal */ @@ -7023,64 +7792,90 @@ yymsp[0].minor.yy600); } case 714: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==714); case 717: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==717); case 719: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==719); +#line 1003 "sql.y" { yylhsminor.yy600 = yymsp[0].minor.yy600; } +#line 7797 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 489: /* literal ::= NULL */ +#line 1004 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } +#line 7803 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 490: /* literal ::= NK_QUESTION */ +#line 1005 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 7809 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 491: /* duration_literal ::= NK_VARIABLE */ case 688: /* interval_sliding_duration_literal ::= NK_VARIABLE */ yytestcase(yyruleno==688); case 689: /* interval_sliding_duration_literal ::= NK_STRING */ yytestcase(yyruleno==689); case 690: /* interval_sliding_duration_literal ::= NK_INTEGER */ yytestcase(yyruleno==690); +#line 1007 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 7818 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 492: /* signed ::= NK_INTEGER */ +#line 1009 "sql.y" { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } +#line 7824 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 493: /* signed ::= NK_PLUS NK_INTEGER */ +#line 1010 "sql.y" { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } +#line 7830 "sql.c" break; case 494: /* signed ::= NK_MINUS NK_INTEGER */ +#line 1011 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } +#line 7839 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 495: /* signed ::= NK_FLOAT */ +#line 1016 "sql.y" { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +#line 7845 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 496: /* signed ::= NK_PLUS NK_FLOAT */ +#line 1017 "sql.y" { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +#line 7851 "sql.c" break; case 497: /* signed ::= NK_MINUS NK_FLOAT */ +#line 1018 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } +#line 7860 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 499: /* signed_literal ::= NK_STRING */ +#line 1025 "sql.y" { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +#line 7866 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 500: /* signed_literal ::= NK_BOOL */ +#line 1026 "sql.y" { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } +#line 7872 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 501: /* signed_literal ::= TIMESTAMP NK_STRING */ +#line 1027 "sql.y" { yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +#line 7878 "sql.c" break; case 502: /* signed_literal ::= duration_literal */ case 504: /* signed_literal ::= literal_func */ yytestcase(yyruleno==504); @@ -7090,118 +7885,156 @@ yymsp[0].minor.yy600); } case 718: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==718); case 720: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==720); case 733: /* search_condition ::= common_expression */ yytestcase(yyruleno==733); +#line 1028 "sql.y" { yylhsminor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } +#line 7890 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 503: /* signed_literal ::= NULL */ +#line 1029 "sql.y" { yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } +#line 7896 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 505: /* signed_literal ::= NK_QUESTION */ +#line 1031 "sql.y" { yylhsminor.yy600 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } +#line 7902 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 524: /* expression ::= pseudo_column */ +#line 1097 "sql.y" { yylhsminor.yy600 = yymsp[0].minor.yy600; setRawExprNodeIsPseudoColumn(pCxt, yylhsminor.yy600, true); } +#line 7908 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 528: /* expression ::= NK_LP expression NK_RP */ case 616: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==616); case 732: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==732); +#line 1101 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 7916 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 529: /* expression ::= NK_PLUS expr_or_subquery */ +#line 1102 "sql.y" { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 7925 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 530: /* expression ::= NK_MINUS expr_or_subquery */ +#line 1106 "sql.y" { SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); } +#line 7934 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 531: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ +#line 1110 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 7944 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 532: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ +#line 1115 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 7954 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 533: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ +#line 1120 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 7964 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 534: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ +#line 1125 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 7974 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 535: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ +#line 1130 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 7984 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 536: /* expression ::= column_reference NK_ARROW NK_STRING */ +#line 1135 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } +#line 7993 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 537: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ +#line 1139 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8003 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 538: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ +#line 1144 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8013 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 541: /* column_reference ::= column_name */ +#line 1155 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy649, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy649)); } +#line 8019 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 542: /* column_reference ::= table_name NK_DOT column_name */ +#line 1156 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649, createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy649)); } +#line 8025 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 543: /* column_reference ::= NK_ALIAS */ +#line 1157 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } +#line 8031 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 544: /* column_reference ::= table_name NK_DOT NK_ALIAS */ +#line 1158 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0)); } +#line 8037 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 545: /* pseudo_column ::= ROWTS */ @@ -7217,243 +8050,356 @@ yymsp[0].minor.yy600); } case 556: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==556); case 563: /* literal_func ::= NOW */ yytestcase(yyruleno==563); case 564: /* literal_func ::= TODAY */ yytestcase(yyruleno==564); +#line 1160 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } +#line 8055 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 547: /* pseudo_column ::= table_name NK_DOT TBNAME */ +#line 1162 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy649)))); } +#line 8061 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 557: /* function_expression ::= function_name NK_LP expression_list NK_RP */ case 558: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==558); +#line 1173 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy649, yymsp[-1].minor.yy748)); } +#line 8068 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 559: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ case 560: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name_default_len NK_RP */ yytestcase(yyruleno==560); +#line 1176 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy400)); } +#line 8075 "sql.c" yymsp[-5].minor.yy600 = yylhsminor.yy600; break; case 562: /* literal_func ::= noarg_func NK_LP NK_RP */ +#line 1182 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy649, NULL)); } +#line 8081 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 578: /* star_func_para_list ::= NK_STAR */ +#line 1207 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } +#line 8087 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 583: /* star_func_para ::= table_name NK_DOT NK_STAR */ case 668: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==668); +#line 1216 "sql.y" { yylhsminor.yy600 = createColumnNode(pCxt, &yymsp[-2].minor.yy649, &yymsp[0].minor.yy0); } +#line 8094 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 584: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +#line 1219 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy748, yymsp[-1].minor.yy600)); } +#line 8100 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 585: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +#line 1221 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-2].minor.yy748, yymsp[-1].minor.yy600)); } +#line 8106 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 588: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +#line 1228 "sql.y" { yymsp[-3].minor.yy600 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } +#line 8112 "sql.c" break; case 590: /* case_when_else_opt ::= ELSE common_expression */ +#line 1231 "sql.y" { yymsp[-1].minor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } +#line 8117 "sql.c" break; case 591: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ case 596: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==596); +#line 1234 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy292, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8127 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 592: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ +#line 1241 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8137 "sql.c" yymsp[-4].minor.yy600 = yylhsminor.yy600; break; case 593: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ +#line 1247 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8147 "sql.c" yymsp[-5].minor.yy600 = yylhsminor.yy600; break; case 594: /* predicate ::= expr_or_subquery IS NULL */ +#line 1252 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), NULL)); } +#line 8156 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 595: /* predicate ::= expr_or_subquery IS NOT NULL */ +#line 1256 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL)); } +#line 8165 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 597: /* compare_op ::= NK_LT */ +#line 1268 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_LOWER_THAN; } +#line 8171 "sql.c" break; case 598: /* compare_op ::= NK_GT */ +#line 1269 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_GREATER_THAN; } +#line 8176 "sql.c" break; case 599: /* compare_op ::= NK_LE */ +#line 1270 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_LOWER_EQUAL; } +#line 8181 "sql.c" break; case 600: /* compare_op ::= NK_GE */ +#line 1271 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_GREATER_EQUAL; } +#line 8186 "sql.c" break; case 601: /* compare_op ::= NK_NE */ +#line 1272 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_NOT_EQUAL; } +#line 8191 "sql.c" break; case 602: /* compare_op ::= NK_EQ */ +#line 1273 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_EQUAL; } +#line 8196 "sql.c" break; case 603: /* compare_op ::= LIKE */ +#line 1274 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_LIKE; } +#line 8201 "sql.c" break; case 604: /* compare_op ::= NOT LIKE */ +#line 1275 "sql.y" { yymsp[-1].minor.yy292 = OP_TYPE_NOT_LIKE; } +#line 8206 "sql.c" break; case 605: /* compare_op ::= MATCH */ +#line 1276 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_MATCH; } +#line 8211 "sql.c" break; case 606: /* compare_op ::= NMATCH */ +#line 1277 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_NMATCH; } +#line 8216 "sql.c" break; case 607: /* compare_op ::= CONTAINS */ +#line 1278 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_JSON_CONTAINS; } +#line 8221 "sql.c" break; case 608: /* in_op ::= IN */ +#line 1282 "sql.y" { yymsp[0].minor.yy292 = OP_TYPE_IN; } +#line 8226 "sql.c" break; case 609: /* in_op ::= NOT IN */ +#line 1283 "sql.y" { yymsp[-1].minor.yy292 = OP_TYPE_NOT_IN; } +#line 8231 "sql.c" break; case 610: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +#line 1285 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); } +#line 8236 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 612: /* boolean_value_expression ::= NOT boolean_primary */ +#line 1289 "sql.y" { SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); } +#line 8245 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 613: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +#line 1294 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8255 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 614: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +#line 1300 "sql.y" { SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8265 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 622: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +#line 1318 "sql.y" { yylhsminor.yy600 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, JOIN_STYPE_NONE, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, NULL); } +#line 8271 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 625: /* table_primary ::= table_name alias_opt */ +#line 1324 "sql.y" { yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); } +#line 8277 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 626: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +#line 1325 "sql.y" { yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-3].minor.yy649, &yymsp[-1].minor.yy649, &yymsp[0].minor.yy649); } +#line 8283 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 627: /* table_primary ::= subquery alias_opt */ +#line 1326 "sql.y" { yylhsminor.yy600 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy649); } +#line 8289 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 629: /* alias_opt ::= */ +#line 1331 "sql.y" { yymsp[1].minor.yy649 = nil_token; } +#line 8295 "sql.c" break; case 631: /* alias_opt ::= AS table_alias */ +#line 1333 "sql.y" { yymsp[-1].minor.yy649 = yymsp[0].minor.yy649; } +#line 8300 "sql.c" break; case 632: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ case 633: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==633); +#line 1335 "sql.y" { yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; } +#line 8306 "sql.c" break; case 634: /* joined_table ::= table_reference join_type join_subtype JOIN table_reference join_on_clause_opt window_offset_clause_opt jlimit_clause_opt */ +#line 1341 "sql.y" { yylhsminor.yy600 = createJoinTableNode(pCxt, yymsp[-6].minor.yy564, yymsp[-5].minor.yy758, yymsp[-7].minor.yy600, yymsp[-3].minor.yy600, yymsp[-2].minor.yy600); yylhsminor.yy600 = addWindowOffsetClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600); yylhsminor.yy600 = addJLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600); } +#line 8315 "sql.c" yymsp[-7].minor.yy600 = yylhsminor.yy600; break; case 635: /* join_type ::= */ +#line 1349 "sql.y" { yymsp[1].minor.yy564 = JOIN_TYPE_INNER; } +#line 8321 "sql.c" break; case 636: /* join_type ::= INNER */ +#line 1350 "sql.y" { yymsp[0].minor.yy564 = JOIN_TYPE_INNER; } +#line 8326 "sql.c" break; case 637: /* join_type ::= LEFT */ +#line 1351 "sql.y" { yymsp[0].minor.yy564 = JOIN_TYPE_LEFT; } +#line 8331 "sql.c" break; case 638: /* join_type ::= RIGHT */ +#line 1352 "sql.y" { yymsp[0].minor.yy564 = JOIN_TYPE_RIGHT; } +#line 8336 "sql.c" break; case 639: /* join_type ::= FULL */ +#line 1353 "sql.y" { yymsp[0].minor.yy564 = JOIN_TYPE_FULL; } +#line 8341 "sql.c" break; case 640: /* join_subtype ::= */ +#line 1357 "sql.y" { yymsp[1].minor.yy758 = JOIN_STYPE_NONE; } +#line 8346 "sql.c" break; case 641: /* join_subtype ::= OUTER */ +#line 1358 "sql.y" { yymsp[0].minor.yy758 = JOIN_STYPE_OUTER; } +#line 8351 "sql.c" break; case 642: /* join_subtype ::= SEMI */ +#line 1359 "sql.y" { yymsp[0].minor.yy758 = JOIN_STYPE_SEMI; } +#line 8356 "sql.c" break; case 643: /* join_subtype ::= ANTI */ +#line 1360 "sql.y" { yymsp[0].minor.yy758 = JOIN_STYPE_ANTI; } +#line 8361 "sql.c" break; case 644: /* join_subtype ::= ASOF */ +#line 1361 "sql.y" { yymsp[0].minor.yy758 = JOIN_STYPE_ASOF; } +#line 8366 "sql.c" break; case 645: /* join_subtype ::= WINDOW */ +#line 1362 "sql.y" { yymsp[0].minor.yy758 = JOIN_STYPE_WIN; } +#line 8371 "sql.c" break; case 649: /* window_offset_clause_opt ::= WINDOW_OFFSET NK_LP window_offset_literal NK_COMMA window_offset_literal NK_RP */ +#line 1369 "sql.y" { yymsp[-5].minor.yy600 = createWindowOffsetNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 8376 "sql.c" break; case 650: /* window_offset_literal ::= NK_VARIABLE */ +#line 1371 "sql.y" { yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createTimeOffsetValueNode(pCxt, &yymsp[0].minor.yy0)); } +#line 8381 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 651: /* window_offset_literal ::= NK_MINUS NK_VARIABLE */ +#line 1372 "sql.y" { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; yylhsminor.yy600 = createRawExprNode(pCxt, &t, createTimeOffsetValueNode(pCxt, &t)); } +#line 8391 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 653: /* jlimit_clause_opt ::= JLIMIT NK_INTEGER */ case 724: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ yytestcase(yyruleno==724); case 728: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==728); +#line 1379 "sql.y" { yymsp[-1].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } +#line 8399 "sql.c" break; case 654: /* query_specification ::= SELECT hint_list set_quantifier_opt tag_mode_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ +#line 1385 "sql.y" { yymsp[-13].minor.yy600 = createSelectStmt(pCxt, yymsp[-11].minor.yy705, yymsp[-9].minor.yy748, yymsp[-8].minor.yy600, yymsp[-12].minor.yy748); yymsp[-13].minor.yy600 = setSelectStmtTagMode(pCxt, yymsp[-13].minor.yy600, yymsp[-10].minor.yy705); @@ -7466,158 +8412,245 @@ yymsp[0].minor.yy600); } yymsp[-13].minor.yy600 = addEveryClause(pCxt, yymsp[-13].minor.yy600, yymsp[-4].minor.yy600); yymsp[-13].minor.yy600 = addFillClause(pCxt, yymsp[-13].minor.yy600, yymsp[-3].minor.yy600); } +#line 8415 "sql.c" break; case 655: /* hint_list ::= */ +#line 1400 "sql.y" { yymsp[1].minor.yy748 = createHintNodeList(pCxt, NULL); } +#line 8420 "sql.c" break; case 656: /* hint_list ::= NK_HINT */ +#line 1401 "sql.y" { yylhsminor.yy748 = createHintNodeList(pCxt, &yymsp[0].minor.yy0); } +#line 8425 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 661: /* set_quantifier_opt ::= ALL */ +#line 1412 "sql.y" { yymsp[0].minor.yy705 = false; } +#line 8431 "sql.c" break; case 664: /* select_item ::= NK_STAR */ +#line 1419 "sql.y" { yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } +#line 8436 "sql.c" yymsp[0].minor.yy600 = yylhsminor.yy600; break; case 666: /* select_item ::= common_expression column_alias */ case 676: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==676); +#line 1421 "sql.y" { yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy649); } +#line 8443 "sql.c" yymsp[-1].minor.yy600 = yylhsminor.yy600; break; case 667: /* select_item ::= common_expression AS column_alias */ case 677: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==677); +#line 1422 "sql.y" { yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), &yymsp[0].minor.yy649); } +#line 8450 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 672: /* partition_by_clause_opt ::= PARTITION BY partition_list */ case 702: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==702); case 722: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==722); +#line 1431 "sql.y" { yymsp[-2].minor.yy748 = yymsp[0].minor.yy748; } +#line 8458 "sql.c" break; case 679: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA interval_sliding_duration_literal NK_RP */ +#line 1444 "sql.y" { yymsp[-5].minor.yy600 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 8463 "sql.c" break; case 680: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +#line 1445 "sql.y" { yymsp[-3].minor.yy600 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 8468 "sql.c" break; case 681: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ +#line 1447 "sql.y" { yymsp[-5].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 8473 "sql.c" break; case 682: /* twindow_clause_opt ::= INTERVAL NK_LP interval_sliding_duration_literal NK_COMMA interval_sliding_duration_literal NK_RP sliding_opt fill_opt */ +#line 1451 "sql.y" { yymsp[-7].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } +#line 8478 "sql.c" break; case 683: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +#line 1453 "sql.y" { yymsp[-6].minor.yy600 = createEventWindowNode(pCxt, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } +#line 8483 "sql.c" break; case 684: /* twindow_clause_opt ::= COUNT_WINDOW NK_LP NK_INTEGER NK_RP */ +#line 1455 "sql.y" { yymsp[-3].minor.yy600 = createCountWindowNode(pCxt, &yymsp[-1].minor.yy0, &yymsp[-1].minor.yy0); } +#line 8488 "sql.c" break; case 685: /* twindow_clause_opt ::= COUNT_WINDOW NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +#line 1457 "sql.y" { yymsp[-5].minor.yy600 = createCountWindowNode(pCxt, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0); } +#line 8493 "sql.c" break; case 692: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +#line 1467 "sql.y" { yymsp[-3].minor.yy600 = createFillNode(pCxt, yymsp[-1].minor.yy6, NULL); } +#line 8498 "sql.c" break; case 693: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ +#line 1468 "sql.y" { yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); } +#line 8503 "sql.c" break; case 694: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ +#line 1469 "sql.y" { yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy748)); } +#line 8508 "sql.c" break; case 695: /* fill_mode ::= NONE */ +#line 1473 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_NONE; } +#line 8513 "sql.c" break; case 696: /* fill_mode ::= PREV */ +#line 1474 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_PREV; } +#line 8518 "sql.c" break; case 697: /* fill_mode ::= NULL */ +#line 1475 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_NULL; } +#line 8523 "sql.c" break; case 698: /* fill_mode ::= NULL_F */ +#line 1476 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_NULL_F; } +#line 8528 "sql.c" break; case 699: /* fill_mode ::= LINEAR */ +#line 1477 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_LINEAR; } +#line 8533 "sql.c" break; case 700: /* fill_mode ::= NEXT */ +#line 1478 "sql.y" { yymsp[0].minor.yy6 = FILL_MODE_NEXT; } +#line 8538 "sql.c" break; case 703: /* group_by_list ::= expr_or_subquery */ +#line 1487 "sql.y" { yylhsminor.yy748 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8543 "sql.c" yymsp[0].minor.yy748 = yylhsminor.yy748; break; case 704: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +#line 1488 "sql.y" { yylhsminor.yy748 = addNodeToList(pCxt, yymsp[-2].minor.yy748, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } +#line 8549 "sql.c" yymsp[-2].minor.yy748 = yylhsminor.yy748; break; case 708: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +#line 1495 "sql.y" { yymsp[-5].minor.yy600 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 8555 "sql.c" break; case 709: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_RP */ +#line 1497 "sql.y" { yymsp[-3].minor.yy600 = createInterpTimePoint(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } +#line 8560 "sql.c" break; case 712: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ +#line 1504 "sql.y" { yylhsminor.yy600 = addOrderByClause(pCxt, yymsp[-3].minor.yy600, yymsp[-2].minor.yy748); yylhsminor.yy600 = addSlimitClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600); yylhsminor.yy600 = addLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600); } +#line 8569 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 715: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +#line 1514 "sql.y" { yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } +#line 8575 "sql.c" yymsp[-3].minor.yy600 = yylhsminor.yy600; break; case 716: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +#line 1516 "sql.y" { yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } +#line 8581 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 725: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ case 729: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==729); +#line 1531 "sql.y" { yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } +#line 8588 "sql.c" break; case 726: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ case 730: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==730); +#line 1532 "sql.y" { yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } +#line 8594 "sql.c" break; case 731: /* subquery ::= NK_LP query_expression NK_RP */ +#line 1540 "sql.y" { yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy600); } +#line 8599 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 736: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +#line 1554 "sql.y" { yylhsminor.yy600 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), yymsp[-1].minor.yy1010, yymsp[0].minor.yy273); } +#line 8605 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 737: /* ordering_specification_opt ::= */ +#line 1558 "sql.y" { yymsp[1].minor.yy1010 = ORDER_ASC; } +#line 8611 "sql.c" break; case 738: /* ordering_specification_opt ::= ASC */ +#line 1559 "sql.y" { yymsp[0].minor.yy1010 = ORDER_ASC; } +#line 8616 "sql.c" break; case 739: /* ordering_specification_opt ::= DESC */ +#line 1560 "sql.y" { yymsp[0].minor.yy1010 = ORDER_DESC; } +#line 8621 "sql.c" break; case 740: /* null_ordering_opt ::= */ +#line 1564 "sql.y" { yymsp[1].minor.yy273 = NULL_ORDER_DEFAULT; } +#line 8626 "sql.c" break; case 741: /* null_ordering_opt ::= NULLS FIRST */ +#line 1565 "sql.y" { yymsp[-1].minor.yy273 = NULL_ORDER_FIRST; } +#line 8631 "sql.c" break; case 742: /* null_ordering_opt ::= NULLS LAST */ +#line 1566 "sql.y" { yymsp[-1].minor.yy273 = NULL_ORDER_LAST; } +#line 8636 "sql.c" break; case 745: /* column_options ::= column_options ENCODE NK_STRING */ +#line 1574 "sql.y" { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_ENCODE, &yymsp[0].minor.yy0); } +#line 8641 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 746: /* column_options ::= column_options COMPRESS NK_STRING */ +#line 1575 "sql.y" { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_COMPRESS, &yymsp[0].minor.yy0); } +#line 8647 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; case 747: /* column_options ::= column_options LEVEL NK_STRING */ +#line 1576 "sql.y" { yylhsminor.yy600 = setColumnOptions(pCxt, yymsp[-2].minor.yy600, COLUMN_OPTION_LEVEL, &yymsp[0].minor.yy0); } +#line 8653 "sql.c" yymsp[-2].minor.yy600 = yylhsminor.yy600; break; default: @@ -7680,6 +8713,7 @@ static void yy_syntax_error( ParseCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ +#line 29 "sql.y" if (TSDB_CODE_SUCCESS == pCxt->errCode) { if(TOKEN.z) { @@ -7690,6 +8724,7 @@ static void yy_syntax_error( } else if (TSDB_CODE_PAR_DB_NOT_SPECIFIED == pCxt->errCode && TK_NK_FLOAT == TOKEN.type) { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, TOKEN.z); } +#line 8727 "sql.c" /************ End %syntax_error code ******************************************/ ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ ParseCTX_STORE diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 84be1e92d0..4e7325c9b2 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -335,7 +335,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { run("ALTER DATABASE test KEEP 1000000000s", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test KEEP 1w", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test PAGES 63", TSDB_CODE_PAR_INVALID_DB_OPTION); - run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); + //run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION); //run("ALTER DATABASE test REPLICA 2", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test STT_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 23b8baf031..acc29997a2 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1587,6 +1587,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator TSWAP(pProject->node.pLimit, pSetOperator->pLimit); } pProject->ignoreGroupId = true; + pProject->isSetOpProj = true; int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index a2a0343316..b271c83678 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -454,7 +454,12 @@ static int32_t scanPathOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub scanPathOptSetGroupOrderScan(info.pScan); } if (TSDB_CODE_SUCCESS == code && (NULL != info.pDsoFuncs || NULL != info.pSdrFuncs)) { - info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); + if (pCxt->pPlanCxt->streamQuery) { + info.pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; // always load all data for stream query + } else { + info.pScan->dataRequired = scanPathOptGetDataRequired(info.pSdrFuncs); + } + info.pScan->pDynamicScanFuncs = info.pDsoFuncs; } if (TSDB_CODE_SUCCESS == code && info.pScan) { @@ -3284,18 +3289,34 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* SNodeList* pNewChildTargets = nodesMakeList(); if (NULL == pProjectNode->node.pParent) { - SNode* pProjection = NULL; - FOREACH(pProjection, pProjectNode->pProjections) { - SNode* pChildTarget = NULL; - FOREACH(pChildTarget, pChild->pTargets) { - if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { - nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + SNode *pProjection = NULL, *pChildTarget = NULL; + bool needOrderMatch = QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pChild) && ((SProjectLogicNode*)pChild)->isSetOpProj; + bool orderMatch = true; + if (needOrderMatch) { + // For sql: select ... from (select ... union all select ...); + // When eliminating the outer proj (the outer select), we have to make sure that the outer proj projections and + // union all project targets have same columns in the same order. See detail in TD-30188 + FORBOTH(pProjection, pProjectNode->pProjections, pChildTarget, pChild->pTargets) { + if (!pProjection) break; + if (0 != strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + orderMatch = false; break; } + nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + } + } else { + FOREACH(pProjection, pProjectNode->pProjections) { + FOREACH(pChildTarget, pChild->pTargets) { + if (0 == strcmp(((SColumnNode*)pProjection)->colName, ((SColumnNode*)pChildTarget)->colName)) { + nodesListAppend(pNewChildTargets, nodesCloneNode(pChildTarget)); + break; + } + } } } - if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets)) { + if (eliminateProjOptCanChildConditionUseChildTargets(pChild, pNewChildTargets) && + (!needOrderMatch || (needOrderMatch && orderMatch))) { nodesDestroyList(pChild->pTargets); pChild->pTargets = pNewChildTargets; } else { @@ -4143,8 +4164,10 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, pLastRowCols, false, cxt.pkBytes); lastRowScanOptRemoveUslessTargets(pScan->node.pTargets, cxt.pLastCols, cxt.pOtherCols, pLastRowCols); - if (pPKTsCol && ((pScan->node.pTargets->length == 1) || (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) { - // when select last(ts),ts from ..., we add another ts to targets + if (pPKTsCol && + ((cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pPKTsCol, nodesListGetNode(cxt.pLastCols, 0))) || + (pScan->node.pTargets->length == 2 && cxt.pkBytes > 0))) { + // when select last(ts),tbname,ts from ..., we add another ts to targets sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol); nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol)); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 527df395cd..d30b100147 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -372,12 +372,12 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + QW_ERR_RET(TSDB_CODE_GRANT_VIEW_EXPIRED); } if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + QW_ERR_RET(TSDB_CODE_GRANT_AUDIT_EXPIRED); } } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 5f43ae9f3c..50de5e760d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1213,6 +1213,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { res->translate = true; + strcpy(res->node.aliasName, node->node.aliasName); res->node.resType.type = output.columnData->info.type; res->node.resType.bytes = output.columnData->info.bytes; res->node.resType.scale = output.columnData->info.scale; @@ -1268,6 +1269,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { res->node.resType = node->node.resType; res->translate = true; + strcpy(res->node.aliasName, node->node.aliasName); int32_t type = output.columnData->info.type; if (IS_VAR_DATA_TYPE(type)) { res->datum.p = output.columnData->pData; @@ -1309,6 +1311,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { res->translate = true; + strcpy(res->node.aliasName, node->node.aliasName); res->node.resType = node->node.resType; if (colDataIsNull_s(output.columnData, 0)) { res->isNull = true; @@ -1364,6 +1367,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { res->translate = true; + strcpy(res->node.aliasName, node->node.aliasName); res->node.resType = node->node.resType; if (colDataIsNull_s(output.columnData, 0)) { res->isNull = true; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index c52a8599a0..6d2215264e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -798,6 +798,7 @@ TEST(queryTest, normalCase) { schedulerFreeJob(&job, 0); + taosThreadJoin(thread1, NULL); } TEST(queryTest, readyFirstCase) { @@ -907,6 +908,8 @@ TEST(queryTest, readyFirstCase) { schedulerDestroy(); schedulerFreeJob(&job, 0); + + taosThreadJoin(thread1, NULL); } TEST(queryTest, flowCtrlCase) { @@ -1001,6 +1004,8 @@ TEST(queryTest, flowCtrlCase) { schedulerDestroy(); schedulerFreeJob(&job, 0); + + taosThreadJoin(thread1, NULL); } TEST(insertTest, normalCase) { @@ -1061,6 +1066,8 @@ TEST(insertTest, normalCase) { schedulerFreeJob(&insertJobRefId, 0); schedulerDestroy(); + + taosThreadJoin(thread1, NULL); } TEST(multiThread, forceFree) { diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 704bc9a2f2..35114a1be6 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -141,7 +141,7 @@ SListNode* streamBackendAddCompare(void* backend, void* arg); void streamBackendDelCompare(void* backend, void* arg); int32_t streamStateCvtDataFormat(char* path, char* key, void* cfInst); -STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId); +STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId); void taskDbDestroy(void* pBackend, bool flush); void taskDbDestroy2(void* pBackend); int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); @@ -247,6 +247,7 @@ int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId); int32_t streamBackendDelInUseChkp(void* arg, int64_t chkpId); int32_t taskDbBuildSnap(void* arg, SArray* pSnap); +int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo); int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 3ccb25a62a..f3ec01cf7a 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -52,6 +52,23 @@ extern "C" { #define stTrace(...) do { if (stDebugFlag & DEBUG_TRACE) { taosPrintLog("STM ", DEBUG_TRACE, stDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +struct SActiveCheckpointInfo { + TdThreadMutex lock; + int32_t transId; + int64_t firstRecvTs; // first time to recv checkpoint trigger info + int64_t activeId; // current active checkpoint id + int64_t failedId; + bool dispatchTrigger; + SArray* pDispatchTriggerList; // SArray + SArray* pReadyMsgList; // SArray + int8_t allUpstreamTriggerRecv; + SArray* pCheckpointReadyRecvList; // SArray + int32_t checkCounter; + tmr_h pChkptTriggerTmr; + int32_t sendReadyCheckCounter; + tmr_h pSendReadyMsgTmr; +}; + typedef struct { int8_t type; SSDataBlock* pBlock; @@ -81,6 +98,35 @@ struct STokenBucket { int64_t quotaFillTimestamp; // fill timestamp }; +typedef struct { + int32_t upstreamTaskId; + SEpSet upstreamNodeEpset; + int32_t upstreamNodeId; + int32_t transId; + int32_t childId; + SRpcMsg msg; // for mnode checkpoint-source rsp + int64_t checkpointId; + int64_t recvTs; + int32_t sendCompleted; +} STaskCheckpointReadyInfo; + +typedef struct { + int64_t sendTs; + int64_t recvTs; + bool recved; + int32_t nodeId; + int32_t taskId; +} STaskTriggerSendInfo; + +typedef struct { + int64_t streamId; + int64_t recvTs; + int32_t downstreamNodeId; + int32_t downstreamTaskId; + int64_t checkpointId; + int32_t transId; +} STaskDownstreamReadyInfo; + struct SStreamQueue { STaosQueue* pQueue; STaosQall* qall; @@ -113,7 +159,7 @@ void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); int32_t getNumOfDispatchBranch(SStreamTask* pTask); void clearBufferedDispatchMsg(SStreamTask* pTask); -int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); +int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t blockType, int32_t srcVg); SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamTask* pTask, int64_t resultSize, SArray* pRes); @@ -122,7 +168,6 @@ void destroyStreamDataBlock(SStreamDataBlock* pBlock); int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock); -int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); @@ -132,11 +177,13 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask); void streamTaskSetFailedCheckpointId(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); +int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask); int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); STaskId streamTaskGetTaskId(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); int32_t streamTaskResetTimewindowFilter(SStreamTask* pTask); +void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo); void streamClearChkptReadyMsg(SStreamTask* pTask); EExtractDataCode streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInput, int32_t* numOfBlocks, @@ -165,13 +212,20 @@ typedef enum ECHECKPOINT_BACKUP_TYPE { ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType(); -int32_t streamTaskDownloadCheckpointData(char* id, char* path); +int32_t streamTaskDownloadCheckpointData(const char* id, char* path); int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask); -int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask); +int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask); + +int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, + int32_t childId, SEpSet* pEpset, int64_t checkpointId); +int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, + int64_t checkpointId, SRpcMsg* pMsg); typedef int32_t (*__stream_async_exec_fn_t)(void* param); int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code); +void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index db13da5f3b..597d6035d6 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -40,9 +40,9 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, char** cfs, int32_t void destroyRocksdbCfInst(RocksdbCfInst* inst); int32_t getCfIdx(const char* cfName); -STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath); +STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath); -int32_t backendCopyFiles(char* src, char* dst); +static int32_t backendCopyFiles(const char* src, const char* dst); void destroyCompactFilteFactory(void* arg); void destroyCompactFilte(void* arg); @@ -146,6 +146,9 @@ static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, static rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfName, rocksdb_snapshot_t** snapshot, rocksdb_readoptions_t** readOpt); +void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp); +void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp); + #define GEN_COLUMN_FAMILY_NAME(name, idstr, SUFFIX) sprintf(name, "%s_%s", idstr, (SUFFIX)); int32_t copyFiles(const char* src, const char* dst); uint32_t nextPow2(uint32_t x); @@ -192,12 +195,6 @@ int32_t getCfIdx(const char* cfName) { bool isValidCheckpoint(const char* dir) { return true; - STaskDbWrapper* pDb = taskDbOpenImpl(NULL, NULL, (char*)dir); - if (pDb == NULL) { - return false; - } - taskDbDestroy(pDb, false); - return true; } int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { @@ -234,12 +231,14 @@ int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { tstrerror(TAOS_SYSTEM_ERROR(errno)), state); taosMkDir(state); } + taosMemoryFree(chkp); } - *dst = state; + *dst = state; return 0; } + int32_t remoteChkp_readMetaData(char* path, SArray* list) { char* metaPath = taosMemoryCalloc(1, strlen(path)); sprintf(metaPath, "%s%s%s", path, TD_DIRSEP, "META"); @@ -323,27 +322,30 @@ int32_t remoteChkp_validAndCvtMeta(char* path, SArray* list, int64_t chkpId) { return complete == 1 ? 0 : -1; } -int32_t rebuildFromRemoteChkp_rsync(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { - // impl later +int32_t rebuildFromRemoteChkp_rsync(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) { int32_t code = 0; - if (taosIsDir(chkpPath)) { - taosRemoveDir(chkpPath); + if (taosIsDir(chkptPath)) { + taosRemoveDir(chkptPath); + stDebug("remove local checkpoint data dir:%s succ", chkptPath); } if (taosIsDir(defaultPath)) { taosRemoveDir(defaultPath); + taosMulMkDir(defaultPath); + stDebug("clear local default dir before downloading checkpoint data:%s succ", defaultPath); } - code = streamTaskDownloadCheckpointData(key, chkpPath); + code = streamTaskDownloadCheckpointData(key, chkptPath); if (code != 0) { + stError("failed to download checkpoint data:%s", key); return code; } - code = backendCopyFiles(chkpPath, defaultPath); - return code; + stDebug("download remote checkpoint data for checkpointId:%" PRId64 ", %s", checkpointId, key); + return backendCopyFiles(chkptPath, defaultPath); } -int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { +int32_t rebuildFromRemoteChkp_s3(const char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { int32_t code = streamTaskDownloadCheckpointData(key, chkpPath); if (code != 0) { return code; @@ -378,13 +380,16 @@ int32_t rebuildFromRemoteChkp_s3(char* key, char* chkpPath, int64_t chkpId, char return code; } -int32_t rebuildFromRemoteChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { +int32_t rebuildFromRemoteCheckpoint(const char* key, char* chkptPath, int64_t checkpointId, char* defaultPath) { ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType(); if (type == DATA_UPLOAD_S3) { - return rebuildFromRemoteChkp_s3(key, chkpPath, chkpId, defaultPath); + return rebuildFromRemoteChkp_s3(key, chkptPath, checkpointId, defaultPath); } else if (type == DATA_UPLOAD_RSYNC) { - return rebuildFromRemoteChkp_rsync(key, chkpPath, chkpId, defaultPath); + return rebuildFromRemoteChkp_rsync(key, chkptPath, checkpointId, defaultPath); + } else { + stError("%s no remote backup checkpoint data for:%" PRId64, key, checkpointId); } + return -1; } @@ -403,7 +408,7 @@ int32_t copyFiles_hardlink(char* src, char* dst, int8_t type) { return taosLinkFile(src, dst); } -int32_t backendFileCopyFilesImpl(char* src, char* dst) { +int32_t backendFileCopyFilesImpl(const char* src, const char* dst) { const char* current = "CURRENT"; size_t currLen = strlen(current); @@ -412,20 +417,26 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) { int32_t dLen = strlen(dst); char* srcName = taosMemoryCalloc(1, sLen + 64); char* dstName = taosMemoryCalloc(1, dLen + 64); - // copy file to dst + // copy file to dst TdDirPtr pDir = taosOpenDir(src); if (pDir == NULL) { taosMemoryFree(srcName); taosMemoryFree(dstName); + code = TAOS_SYSTEM_ERROR(errno); + errno = 0; - return -1; + return code; } + errno = 0; TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { char* name = taosGetDirEntryName(de); - if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) { + continue; + } sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); @@ -433,18 +444,21 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) { if (strncmp(name, current, strlen(name) <= currLen ? strlen(name) : currLen) == 0) { code = copyFiles_create(srcName, dstName, 0); if (code != 0) { - stError("failed to copy file, detail: %s to %s reason: %s", srcName, dstName, - tstrerror(TAOS_SYSTEM_ERROR(code))); + code = TAOS_SYSTEM_ERROR(code); + stError("failed to copy file, detail: %s to %s reason:%s", srcName, dstName, tstrerror(code)); goto _ERROR; } } else { code = copyFiles_hardlink(srcName, dstName, 0); if (code != 0) { - stError("failed to hard line file, detail: %s to %s, reason: %s", srcName, dstName, - tstrerror(TAOS_SYSTEM_ERROR(code))); + code = TAOS_SYSTEM_ERROR(code); + stError("failed to hard link file, detail:%s to %s, reason:%s", srcName, dstName, tstrerror(code)); goto _ERROR; + } else { + stDebug("succ hard link file:%s to %s", srcName, dstName); } } + memset(srcName, 0, sLen + 64); memset(dstName, 0, dLen + 64); } @@ -453,88 +467,54 @@ int32_t backendFileCopyFilesImpl(char* src, char* dst) { taosMemoryFreeClear(dstName); taosCloseDir(&pDir); errno = 0; - return 0; + return code; + _ERROR: taosMemoryFreeClear(srcName); taosMemoryFreeClear(dstName); taosCloseDir(&pDir); errno = 0; - return -1; + return code; } -int32_t backendCopyFiles(char* src, char* dst) { + +int32_t backendCopyFiles(const char* src, const char* dst) { return backendFileCopyFilesImpl(src, dst); - // // opt later, just hard link - // int32_t sLen = strlen(src); - // int32_t dLen = strlen(dst); - // char* srcName = taosMemoryCalloc(1, sLen + 64); - // char* dstName = taosMemoryCalloc(1, dLen + 64); - - // TdDirPtr pDir = taosOpenDir(src); - // if (pDir == NULL) { - // taosMemoryFree(srcName); - // taosMemoryFree(dstName); - // return -1; - // } - - // TdDirEntryPtr de = NULL; - // while ((de = taosReadDir(pDir)) != NULL) { - // char* name = taosGetDirEntryName(de); - // if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; - - // sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name); - // sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name); - // // if (!taosDirEntryIsDir(de)) { - // // // code = taosCopyFile(srcName, dstName); - // // if (code == -1) { - // // goto _err; - // // } - // // } - // return backendFileCopyFilesImpl(src, dst); - - // memset(srcName, 0, sLen + 64); - // memset(dstName, 0, dLen + 64); - // } - - // _err: - // taosMemoryFreeClear(srcName); - // taosMemoryFreeClear(dstName); - // taosCloseDir(&pDir); - // return code >= 0 ? 0 : -1; - - // return 0; } -int32_t rebuildFromLocalChkp(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { + +static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId, + const char* defaultPath) { int32_t code = 0; + if (taosIsDir(defaultPath)) { taosRemoveDir(defaultPath); taosMkDir(defaultPath); - - stInfo("succ to clear stream backend %s", defaultPath); + stInfo("%s clear local backend dir:%s, succ", pTaskIdStr, defaultPath); } - if (taosIsDir(chkpPath) && isValidCheckpoint(chkpPath)) { - code = backendCopyFiles(chkpPath, defaultPath); - if (code != 0) { + + if (taosIsDir(checkpointPath) && isValidCheckpoint(checkpointPath)) { + stDebug("%s local checkpoint data existed, checkpointId:%" PRId64 " copy to backend dir", pTaskIdStr, checkpointId); + + code = backendCopyFiles(checkpointPath, defaultPath); + if (code != TSDB_CODE_SUCCESS) { taosRemoveDir(defaultPath); taosMkDir(defaultPath); - stError("failed to restart stream backend from %s, reason: %s, start to restart from empty path: %s", chkpPath, - tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); - code = 0; + stError("%s failed to start stream backend from local %s, reason:%s, try download checkpoint from remote", + pTaskIdStr, checkpointPath, tstrerror(TAOS_SYSTEM_ERROR(errno))); + code = TSDB_CODE_SUCCESS; } else { - stInfo("start to restart stream backend at checkpoint path: %s", chkpPath); + stInfo("%s copy checkpoint data from:%s to:%s succ, try to start stream backend", pTaskIdStr, checkpointPath, + defaultPath); } + } else { + code = TSDB_CODE_FAILED; + stError("%s no valid data for checkpointId:%" PRId64 " in %s", pTaskIdStr, checkpointId, checkpointPath); } return code; } -int32_t rebuildFromlocalDefault(char* key, char* chkpPath, int64_t chkpId, char* defaultPath) { - int32_t code = 0; - return code; -} - -int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** dbPrefixPath, char** dbPath) { - // impl later +int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId, char** dbPrefixPath, char** dbPath) { int32_t code = 0; char* prefixPath = taosMemoryCalloc(1, strlen(path) + 128); @@ -547,33 +527,42 @@ int32_t rebuildDirFromChkp2(const char* path, char* key, int64_t chkpId, char** char* defaultPath = taosMemoryCalloc(1, strlen(path) + 256); sprintf(defaultPath, "%s%s%s", prefixPath, TD_DIRSEP, "state"); + if (!taosIsDir(defaultPath)) { taosMulMkDir(defaultPath); } - char* chkpPath = taosMemoryCalloc(1, strlen(path) + 256); - if (chkpId != 0) { - sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkpId); - code = rebuildFromLocalChkp(key, chkpPath, chkpId, defaultPath); - if (code != 0) { - code = rebuildFromRemoteChkp(key, chkpPath, chkpId, defaultPath); - } + int32_t pathLen = strlen(path) + 256; - if (code != 0) { - stInfo("failed to start stream backend at %s, reason: %s, restart from default defaultPath dir:%s", chkpPath, - tstrerror(TAOS_SYSTEM_ERROR(errno)), defaultPath); - code = taosMkDir(defaultPath); - } - } else { - sprintf(chkpPath, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", - (int64_t)-1); + char* checkpointRoot = taosMemoryCalloc(1, pathLen); + sprintf(checkpointRoot, "%s%s%s", prefixPath, TD_DIRSEP, "checkpoints"); - code = rebuildFromLocalChkp(key, chkpPath, -1, defaultPath); - if (code != 0) { - code = taosMkDir(defaultPath); - } + if (!taosIsDir(checkpointRoot)) { + taosMulMkDir(checkpointRoot); } - taosMemoryFree(chkpPath); + taosMemoryFree(checkpointRoot); + + stDebug("%s check local backend dir:%s, checkpointId:%" PRId64 " succ", key, defaultPath, chkptId); + + char* chkptPath = taosMemoryCalloc(1, pathLen); + if (chkptId > 0) { + snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId); + + code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath); + if (code != 0) { + code = rebuildFromRemoteCheckpoint(key, chkptPath, chkptId, defaultPath); + } + + if (code != 0) { + stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath, + tstrerror(code), defaultPath); + code = 0; // reset the error code + } + } else { // no valid checkpoint id + stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data", key); + } + + taosMemoryFree(chkptPath); *dbPath = defaultPath; *dbPrefixPath = prefixPath; @@ -591,11 +580,12 @@ bool streamBackendDataIsExist(const char* path, int64_t chkpId, int32_t vgId) { taosMemoryFree(state); return exist; } + void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { char* backendPath = NULL; int32_t code = rebuildDirFromCheckpoint(streamPath, chkpId, &backendPath); - stDebug("start to init stream backend at %s, checkpointid: %" PRId64 " vgId:%d", backendPath, chkpId, vgId); + stDebug("start to init stream backend:%s, checkpointId:%" PRId64 " vgId:%d", backendPath, chkpId, vgId); uint32_t dbMemLimit = nextPow2(tsMaxStreamBackendCache) << 20; SBackendWrapper* pHandle = taosMemoryCalloc(1, sizeof(SBackendWrapper)); @@ -654,10 +644,12 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { goto _EXIT; } } + if (cfs != NULL) { rocksdb_list_column_families_destroy(cfs, nCf); } - stDebug("succ to init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId); + + stDebug("init stream backend at %s, backend:%p, vgId:%d", backendPath, pHandle, vgId); taosMemoryFreeClear(backendPath); return (void*)pHandle; @@ -1010,6 +1002,50 @@ int32_t chkpGetAllDbCfHandle(SStreamMeta* pMeta, rocksdb_column_family_handle_t* } #endif +int chkpIdComp(const void* a, const void* b) { + int64_t x = *(int64_t*)a; + int64_t y = *(int64_t*)b; + if (x == y) return 0; + + return x < y ? -1 : 1; +} +int32_t taskDbLoadChkpInfo(STaskDbWrapper* pBackend) { + int32_t code = 0; + char* pChkpDir = taosMemoryCalloc(1, 256); + + sprintf(pChkpDir, "%s%s%s", pBackend->path, TD_DIRSEP, "checkpoints"); + if (!taosIsDir(pChkpDir)) { + taosMemoryFree(pChkpDir); + return 0; + } + TdDirPtr pDir = taosOpenDir(pChkpDir); + if (pDir == NULL) { + taosMemoryFree(pChkpDir); + return 0; + } + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; + + if (taosDirEntryIsDir(de)) { + char checkpointPrefix[32] = {0}; + int64_t checkpointId = 0; + + int ret = sscanf(taosGetDirEntryName(de), "checkpoint%" PRId64 "", &checkpointId); + if (ret == 1) { + taosArrayPush(pBackend->chkpSaved, &checkpointId); + } + } else { + continue; + } + } + taosArraySort(pBackend->chkpSaved, chkpIdComp); + + taosMemoryFree(pChkpDir); + taosCloseDir(&pDir); + + return 0; +} int32_t chkpGetAllDbCfHandle2(STaskDbWrapper* pBackend, rocksdb_column_family_handle_t*** ppHandle) { SArray* pHandle = taosArrayInit(8, POINTER_BYTES); for (int i = 0; i < sizeof(ginitDict) / sizeof(ginitDict[0]); i++) { @@ -1043,8 +1079,7 @@ int32_t chkpDoDbCheckpoint(rocksdb_t* db, char* path) { taosMemoryFreeClear(err); goto _ERROR; } - - rocksdb_checkpoint_create(cp, path, 64 << 20, &err); + rocksdb_checkpoint_create(cp, path, UINT64_MAX, &err); if (err != NULL) { stError("failed to do checkpoint at:%s, reason:%s", path, err); taosMemoryFreeClear(err); @@ -1055,8 +1090,8 @@ _ERROR: rocksdb_checkpoint_object_destroy(cp); return code; } + int32_t chkpPreFlushDb(rocksdb_t* db, rocksdb_column_family_handle_t** cf, int32_t nCf) { - if (nCf == 0) return 0; int code = 0; char* err = NULL; @@ -1098,6 +1133,7 @@ int32_t chkpPreBuildDir(char* path, int64_t chkpId, char** chkpDir, char** chkpI return 0; } + int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { SStreamMeta* pMeta = arg; @@ -1109,7 +1145,13 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { STaskDbWrapper* pTaskDb = *(STaskDbWrapper**)pIter; taskDbAddRef(pTaskDb); - code = taskDbDoCheckpoint(pTaskDb, pTaskDb->chkpId); + int64_t chkpId = pTaskDb->chkpId; + taskDbRefChkp(pTaskDb, chkpId); + code = taskDbDoCheckpoint(pTaskDb, chkpId); + if (code != 0) { + taskDbUnRefChkp(pTaskDb, chkpId); + } + taskDbRemoveRef(pTaskDb); SStreamTask* pTask = pTaskDb->pTask; @@ -1124,6 +1166,29 @@ int32_t taskDbBuildSnap(void* arg, SArray* pSnap) { return code; } +int32_t taskDbDestroySnap(void* arg, SArray* pSnapInfo) { + if (pSnapInfo == NULL) return 0; + SStreamMeta* pMeta = arg; + int32_t code = 0; + taosThreadMutexLock(&pMeta->backendMutex); + + char buf[128] = {0}; + for (int i = 0; i < taosArrayGetSize(pSnapInfo); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfo, i); + sprintf(buf, "0x%" PRIx64 "-0x%x", pSnap->streamId, (int32_t)pSnap->taskId); + STaskDbWrapper** pTaskDb = taosHashGet(pMeta->pTaskDbUnique, buf, strlen(buf)); + if (pTaskDb == NULL || *pTaskDb == NULL) { + stWarn("stream backend:%p failed to find task db, streamId:% " PRId64 "", pMeta, pSnap->streamId); + memset(buf, 0, sizeof(buf)); + continue; + } + memset(buf, 0, sizeof(buf)); + + taskDbUnRefChkp(*pTaskDb, pSnap->chkpId); + } + taosThreadMutexUnlock(&pMeta->backendMutex); + return 0; +} #ifdef BUILD_NO_CALL int32_t streamBackendAddInUseChkp(void* arg, int64_t chkpId) { // if (arg == NULL) return 0; @@ -1192,20 +1257,23 @@ int32_t taskDbDoCheckpoint(void* arg, int64_t chkpId) { int32_t nCf = chkpGetAllDbCfHandle2(pTaskDb, &ppCf); stDebug("stream backend:%p start to do checkpoint at:%s, cf num: %d ", pTaskDb, pChkpIdDir, nCf); - if ((code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf)) == 0) { - if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { - stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); - } else { - stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, - taosGetTimestampMs() - st); - } + int64_t written = atomic_load_64(&pTaskDb->dataWritten); + + if (written > 0) { + stDebug("stream backend:%p start to flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written); + code = chkpPreFlushDb(pTaskDb->db, ppCf, nCf); } else { - stError("stream backend:%p failed to flush db at:%s", pTaskDb, pChkpIdDir); + stDebug("stream backend:%p not need flush db at:%s, data written:%" PRId64 "", pTaskDb, pChkpIdDir, written); + } + if ((code = chkpDoDbCheckpoint(pTaskDb->db, pChkpIdDir)) != 0) { + stError("stream backend:%p failed to do checkpoint at:%s", pTaskDb, pChkpIdDir); + } else { + stDebug("stream backend:%p end to do checkpoint at:%s, time cost:%" PRId64 "ms", pTaskDb, pChkpIdDir, + taosGetTimestampMs() - st); } code = chkpMayDelObsolete(pTaskDb, chkpId, pChkpDir); - pTaskDb->dataWritten = 0; - + atomic_store_64(&pTaskDb->dataWritten, 0); pTaskDb->chkpId = chkpId; _EXIT: @@ -1215,6 +1283,7 @@ _EXIT: taosMemoryFree(ppCf); return code; } + int32_t streamBackendDoCheckpoint(void* arg, int64_t chkpId) { return taskDbDoCheckpoint(arg, chkpId); } SListNode* streamBackendAddCompare(void* backend, void* arg) { @@ -1846,7 +1915,7 @@ void taskDbInitOpt(STaskDbWrapper* pTaskDb) { rocksdb_options_set_create_if_missing(opts, 1); rocksdb_options_set_create_missing_column_families(opts, 1); // rocksdb_options_set_max_total_wal_size(opts, dbMemLimit); - rocksdb_options_set_recycle_log_file_num(opts, 6); + // rocksdb_options_set_ecycle_log_file_num(opts, 6); rocksdb_options_set_max_write_buffer_number(opts, 3); rocksdb_options_set_info_log_level(opts, 1); rocksdb_options_set_db_write_buffer_size(opts, 256 << 20); @@ -1900,11 +1969,33 @@ void taskDbInitChkpOpt(STaskDbWrapper* pTaskDb) { pTaskDb->chkpId = -1; pTaskDb->chkpCap = 4; pTaskDb->chkpSaved = taosArrayInit(4, sizeof(int64_t)); + taskDbLoadChkpInfo(pTaskDb); + pTaskDb->chkpInUse = taosArrayInit(4, sizeof(int64_t)); taosThreadRwlockInit(&pTaskDb->chkpDirLock, NULL); } +void taskDbRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { + taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); + taosArrayPush(pTaskDb->chkpInUse, &chkp); + taosArraySort(pTaskDb->chkpInUse, chkpIdComp); + taosThreadRwlockUnlock(&pTaskDb->chkpDirLock); +} + +void taskDbUnRefChkp(STaskDbWrapper* pTaskDb, int64_t chkp) { + taosThreadRwlockWrlock(&pTaskDb->chkpDirLock); + int32_t size = taosArrayGetSize(pTaskDb->chkpInUse); + for (int i = 0; i < size; i++) { + int64_t* p = taosArrayGet(pTaskDb->chkpInUse, i); + if (*p == chkp) { + taosArrayRemove(pTaskDb->chkpInUse, i); + break; + } + } + taosThreadRwlockUnlock(&pTaskDb->chkpDirLock); +} + void taskDbDestroyChkpOpt(STaskDbWrapper* pTaskDb) { taosArrayDestroy(pTaskDb->chkpSaved); taosArrayDestroy(pTaskDb->chkpInUse); @@ -1941,6 +2032,7 @@ int32_t taskDbBuildFullPath(char* path, char* key, char** dbFullPath, char** sta *stateFullPath = statePath; return 0; } + void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { STaskDbWrapper* p = pTaskDb; taosThreadMutexLock(&p->mutex); @@ -1948,7 +2040,7 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId) { taosThreadMutexUnlock(&p->mutex); } -STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { +STaskDbWrapper* taskDbOpenImpl(const char* key, char* statePath, char* dbPath) { char* err = NULL; char** cfNames = NULL; size_t nCf = 0; @@ -1963,7 +2055,7 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); if (nCf == 0) { - stInfo("newly create db, need to restart"); + stInfo("%s newly create db, need to restart", key); // pre create db pTaskDb->db = rocksdb_open(pTaskDb->pCfOpts[0], dbPath, &err); if (pTaskDb->db == NULL) goto _EXIT; @@ -1972,11 +2064,15 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { if (cfNames != NULL) { rocksdb_list_column_families_destroy(cfNames, nCf); } + taosMemoryFree(err); err = NULL; cfNames = rocksdb_list_column_families(pTaskDb->dbOpt, dbPath, &nCf, &err); - ASSERT(err == NULL); + if (err != NULL) { + stError("%s failed to create column-family, %s, %" PRIzu ", reason:%s", key, dbPath, nCf, err); + goto _EXIT; + } } if (taskDbOpenCfs(pTaskDb, dbPath, cfNames, nCf) != 0) { @@ -1988,21 +2084,21 @@ STaskDbWrapper* taskDbOpenImpl(char* key, char* statePath, char* dbPath) { cfNames = NULL; } - stDebug("succ to init stream backend at %s, backend:%p", dbPath, pTaskDb); + stDebug("init s-task backend in:%s, backend:%p, %s", dbPath, pTaskDb, key); return pTaskDb; -_EXIT: +_EXIT: taskDbDestroy(pTaskDb, false); if (err) taosMemoryFree(err); if (cfNames) rocksdb_list_column_families_destroy(cfNames, nCf); return NULL; } -STaskDbWrapper* taskDbOpen(char* path, char* key, int64_t chkpId) { +STaskDbWrapper* taskDbOpen(const char* path, const char* key, int64_t chkptId) { char* statePath = NULL; char* dbPath = NULL; - if (rebuildDirFromChkp2(path, key, chkpId, &statePath, &dbPath) != 0) { + if (restoreCheckpointData(path, key, chkptId, &statePath, &dbPath) != 0) { return NULL; } @@ -2131,15 +2227,18 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 return code; } int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list) { + int32_t code = -1; STaskDbWrapper* pDb = arg; ECHECKPOINT_BACKUP_TYPE utype = type; + taskDbRefChkp(pDb, chkpId); if (utype == DATA_UPLOAD_RSYNC) { - return taskDbGenChkpUploadData__rsync(pDb, chkpId, path); + code = taskDbGenChkpUploadData__rsync(pDb, chkpId, path); } else if (utype == DATA_UPLOAD_S3) { - return taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list); + code = taskDbGenChkpUploadData__s3(pDb, mgt, chkpId, path, list); } - return -1; + taskDbUnRefChkp(pDb, chkpId); + return code; } int32_t taskDbOpenCfByKey(STaskDbWrapper* pDb, const char* key) { @@ -2563,7 +2662,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - wrapper->dataWritten += 1; \ + atomic_add_fetch_64(&wrapper->dataWritten, 1); \ char toString[128] = {0}; \ if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ @@ -2640,7 +2739,7 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfKe break; \ } \ STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; \ - wrapper->dataWritten += 1; \ + atomic_add_fetch_64(&wrapper->dataWritten, 1); \ char toString[128] = {0}; \ if (stDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ @@ -2681,7 +2780,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { stDebug("streamStateClear_rocksdb"); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - wrapper->dataWritten += 1; + atomic_add_fetch_64(&wrapper->dataWritten, 1); char sKeyStr[128] = {0}; char eKeyStr[128] = {0}; @@ -3707,7 +3806,7 @@ void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rock int32_t streamStatePutBatch(SStreamState* pState, const char* cfKeyName, rocksdb_writebatch_t* pBatch, void* key, void* val, int32_t vlen, int64_t ttl) { STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - wrapper->dataWritten += 1; + atomic_add_fetch_64(&wrapper->dataWritten, 1); int i = streamStateGetCfIdx(pState, cfKeyName); if (i < 0) { @@ -3741,7 +3840,8 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb int32_t ttlVLen = ginitDict[cfIdx].enValueFunc(val, vlen, ttl, &ttlV); STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - wrapper->dataWritten += 1; + + atomic_add_fetch_64(&wrapper->dataWritten, 1); rocksdb_column_family_handle_t* pCf = wrapper->pCf[ginitDict[cfIdx].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); @@ -3760,7 +3860,7 @@ int32_t streamStatePutBatchOptimize(SStreamState* pState, int32_t cfIdx, rocksdb int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch) { char* err = NULL; STaskDbWrapper* wrapper = pState->pTdbState->pOwner->pBackend; - wrapper->dataWritten += 1; + atomic_add_fetch_64(&wrapper->dataWritten, 1); rocksdb_write(wrapper->db, wrapper->writeOpt, (rocksdb_writebatch_t*)pBatch, &err); if (err != NULL) { stError("streamState failed to write batch, err:%s", err); diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 4a8ca69ba5..11fecf7683 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -40,7 +40,7 @@ static SDownstreamStatusInfo* findCheckRspStatus(STaskCheckInfo* pInfo, int32_t int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { - SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); ASSERT(pInfo != NULL); *oldStage = pInfo->stage; @@ -246,7 +246,7 @@ int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* } int32_t streamTaskSendCheckRsp(const SStreamMeta* pMeta, int32_t vgId, SStreamTaskCheckRsp* pRsp, - SRpcHandleInfo* pRpcInfo, int32_t taskId) { + SRpcHandleInfo* pRpcInfo, int32_t taskId) { SEncoder encoder; int32_t code; int32_t len; @@ -285,7 +285,7 @@ int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask) { streamTaskInitTaskCheckInfo(pInfo, &pTask->outputInfo, taosGetTimestampMs()); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s start check-rsp monit, ref:%d ", pTask->id.idStr, ref); + stDebug("s-task:%s start check-rsp monitor, ref:%d ", pTask->id.idStr, ref); if (pInfo->checkRspTmr == NULL) { pInfo->checkRspTmr = taosTmrStart(rspMonitorFn, CHECK_RSP_CHECK_INTERVAL, pTask, streamTimer); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 5a4e3a5439..ab3b5d6fa0 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -20,9 +20,9 @@ typedef struct { ECHECKPOINT_BACKUP_TYPE type; - char* taskId; - int64_t chkpId; + char* taskId; + int64_t chkpId; SStreamTask* pTask; int64_t dbRefId; void* pMeta; @@ -30,24 +30,40 @@ typedef struct { static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName); static int32_t deleteCheckpointFile(const char* id, const char* name); -static int32_t streamTaskBackupCheckpoint(const char* id, const char* path); +static int32_t streamTaskUploadCheckpoint(const char* id, const char* path); static int32_t deleteCheckpoint(const char* id); static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); +static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask); +static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); +static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); +static void checkpointTriggerMonitorFn(void* param, void* tmrId); -static int32_t streamAlignCheckpoint(SStreamTask* pTask) { - int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); - int64_t old = atomic_val_compare_exchange_32(&pTask->chkInfo.downstreamAlignNum, 0, num); - if (old == 0) { - stDebug("s-task:%s set initial align upstream num:%d", pTask->id.idStr, num); +static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); + +bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) { + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + int32_t numOfUpstreams = taosArrayGetSize(pTask->upstreamInfo.pList); + bool allSend = true; + + taosThreadMutexLock(&pActiveInfo->lock); + int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); + + if (numOfRecv < numOfUpstreams) { + stDebug("s-task:%s received checkpoint-trigger block, idx:%d, %d upstream tasks not send yet, total:%d", + pTask->id.idStr, pTask->info.selfChildId, (numOfUpstreams - numOfRecv), numOfUpstreams); + allSend = false; } - return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1); + taosThreadMutexUnlock(&pActiveInfo->lock); + return allSend; } -static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) { +SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, + int32_t transId) { SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pChkpoint == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } pChkpoint->type = checkpointType; @@ -55,12 +71,13 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { taosFreeQitem(pChkpoint); - return TSDB_CODE_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } pBlock->info.type = STREAM_CHECKPOINT; - pBlock->info.version = pTask->chkInfo.checkpointingId; - pBlock->info.window.ekey = pBlock->info.window.skey = pTask->chkInfo.transId; // NOTE: set the transId + pBlock->info.version = checkpointId; + pBlock->info.window.ekey = pBlock->info.window.skey = transId; // NOTE: set the transId pBlock->info.rows = 1; pBlock->info.childId = pTask->info.selfChildId; @@ -68,7 +85,15 @@ static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpoint taosArrayPush(pChkpoint->blocks, pBlock); taosMemoryFree(pBlock); - if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pChkpoint) < 0) { + terrno = 0; + + return pChkpoint; +} + +int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId) { + SStreamDataBlock* pCheckpoint = createChkptTriggerBlock(pTask, checkpointType, checkpointId, transId); + + if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pCheckpoint) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -84,23 +109,57 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); ASSERT(code == TSDB_CODE_SUCCESS); - pTask->chkInfo.transId = pReq->transId; - pTask->chkInfo.checkpointingId = pReq->checkpointId; - pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask); + pTask->chkInfo.pActiveInfo->transId = pReq->transId; + pTask->chkInfo.pActiveInfo->activeId = pReq->checkpointId; pTask->chkInfo.startTs = taosGetTimestampMs(); pTask->execInfo.checkpoint += 1; // 2. Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task - return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER); + // and this is the last item in the inputQ. + return appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pReq->checkpointId, pReq->transId); } -static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) { +int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) { + ASSERT(pTask->info.taskLevel != TASK_LEVEL__SOURCE); + + if (pRsp->rspCode != TSDB_CODE_SUCCESS) { + stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, + pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); + return TSDB_CODE_SUCCESS; + } + + appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT_TRIGGER, pRsp->checkpointId, pRsp->transId); + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId, SRpcHandleInfo* pRpcInfo, int32_t code) { + SCheckpointTriggerRsp* pRsp = rpcMallocCont(sizeof(SCheckpointTriggerRsp)); + pRsp->streamId = pTask->id.streamId; + pRsp->upstreamTaskId = pTask->id.taskId; + pRsp->taskId = dstTaskId; + + if (code == TSDB_CODE_SUCCESS) { + pRsp->checkpointId = pTask->chkInfo.pActiveInfo->activeId; + pRsp->transId = pTask->chkInfo.pActiveInfo->transId; + } else { + pRsp->checkpointId = -1; + pRsp->transId = -1; + } + + pRsp->rspCode = code; + + SRpcMsg rspMsg = {.code = 0, .pCont = pRsp, .contLen = sizeof(SCheckpointTriggerRsp), .info = *pRpcInfo}; + tmsgSendRsp(&rspMsg); + return 0; +} + +int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask) { pBlock->srcTaskId = pTask->id.taskId; pBlock->srcVgId = pTask->pMeta->vgId; int32_t code = taosWriteQitem(pTask->outputq.queue->pQueue, pBlock); if (code == 0) { - ASSERT(pTask->chkInfo.dispatchCheckpointTrigger == false); + ASSERT(pTask->chkInfo.pActiveInfo->dispatchTrigger == false); streamDispatchStreamBlock(pTask); } else { stError("s-task:%s failed to put checkpoint into outputQ, code:%s", pTask->id.idStr, tstrerror(code)); @@ -110,22 +169,90 @@ static int32_t continueDispatchCheckpointBlock(SStreamDataBlock* pBlock, SStream return code; } -int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { +int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock) { SSDataBlock* pDataBlock = taosArrayGet(pBlock->blocks, 0); int64_t checkpointId = pDataBlock->info.version; int32_t transId = pDataBlock->info.window.skey; const char* id = pTask->id.idStr; int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = pTask->pMeta->vgId; + int32_t taskLevel = pTask->info.taskLevel; - stDebug("s-task:%s vgId:%d start to handle the checkpoint block, checkpointId:%" PRId64 " ver:%" PRId64 - ", transId:%d current checkpointingId:%" PRId64, + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + taosThreadMutexLock(&pTask->lock); + if (pTask->chkInfo.checkpointId > checkpointId) { + stError("s-task:%s vgId:%d current checkpointId:%" PRId64 + " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", + id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; + } + + if (pTask->chkInfo.checkpointId == checkpointId) { + { // send checkpoint-ready msg to upstream + SRpcMsg msg ={0}; + + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId); + initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); + tmsgSendReq(&pInfo->epSet, &msg); + } + + stWarn( + "s-task:%s vgId:%d recv already finished checkpoint msg, send checkpoint-ready to upstream:0x%x to resume the " + "interrupted checkpoint", + id, vgId, pBlock->srcTaskId); + + streamTaskOpenUpstreamInput(pTask, pBlock->srcTaskId); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; + } + + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK) { + if (pActiveInfo->activeId != checkpointId) { + stError("s-task:%s vgId:%d active checkpointId:%" PRId64 ", recv invalid checkpoint-trigger checkpointId:%" PRId64 + " discard", + id, vgId, pActiveInfo->activeId, checkpointId); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; + } else { // checkpointId == pActiveInfo->activeId + if (pActiveInfo->allUpstreamTriggerRecv == 1) { + stDebug( + "s-task:%s vgId:%d all upstream checkpoint-trigger recv, discard this checkpoint-trigger, " + "checkpointId:%" PRId64 " transId:%d", + id, vgId, checkpointId, transId); + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; + } + + if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { + // check if already recv or not, and duplicated checkpoint-trigger msg recv, discard it + for (int32_t i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++i) { + STaskCheckpointReadyInfo* p = taosArrayGet(pActiveInfo->pReadyMsgList, i); + if (p->upstreamTaskId == pBlock->srcTaskId) { + ASSERT(p->checkpointId == checkpointId); + stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 + ", prev recvTs:%" PRId64 " discard", + pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); + + taosThreadMutexUnlock(&pTask->lock); + return TSDB_CODE_SUCCESS; + } + } + } + } + } + + taosThreadMutexUnlock(&pTask->lock); + + stDebug("s-task:%s vgId:%d start to handle the checkpoint-trigger block, checkpointId:%" PRId64 " ver:%" PRId64 + ", transId:%d current active checkpointId:%" PRId64, id, vgId, pTask->chkInfo.checkpointId, pTask->chkInfo.checkpointVer, transId, checkpointId); // set task status if (streamTaskGetStatus(pTask)->state != TASK_STATUS__CK) { - pTask->chkInfo.checkpointingId = checkpointId; - pTask->chkInfo.transId = transId; + pActiveInfo->activeId = checkpointId; + pActiveInfo->transId = transId; code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); if (code != TSDB_CODE_SUCCESS) { @@ -133,18 +260,33 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc streamFreeQitem((SStreamQueueItem*)pBlock); return code; } + + int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s start checkpoint-trigger monitor in 10s, ref:%d ", pTask->id.idStr, ref); + streamMetaAcquireOneTask(pTask); + + if (pActiveInfo->pChkptTriggerTmr == NULL) { + pActiveInfo->pChkptTriggerTmr = taosTmrStart(checkpointTriggerMonitorFn, 100, pTask, streamTimer); + } else { + taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + } } - // todo fix race condition: set the status and append checkpoint block - int32_t taskLevel = pTask->info.taskLevel; if (taskLevel == TASK_LEVEL__SOURCE) { int8_t type = pTask->outputInfo.type; + pActiveInfo->allUpstreamTriggerRecv = 1; + + // We need to transfer state here, before dispatching checkpoint-trigger to downstream tasks. + // The transfer of state may generate new data that need to dispatch to downstream tasks, + // Otherwise, those new generated data by executors that is kept in outputQ, may be lost if this program crashed + // before the next checkpoint. + flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); + if (type == TASK_OUTPUT__FIXED_DISPATCH || type == TASK_OUTPUT__SHUFFLE_DISPATCH) { stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId); - continueDispatchCheckpointBlock(pBlock, pTask); + continueDispatchCheckpointTriggerBlock(pBlock, pTask); } else { // only one task exists, no need to dispatch downstream info - atomic_add_fetch_32(&pTask->chkInfo.numOfNotReady, 1); - streamProcessCheckpointReadyMsg(pTask); + appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId); streamFreeQitem((SStreamQueueItem*)pBlock); } } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { @@ -158,31 +300,24 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc streamAddCheckpointReadyMsg(pTask, pBlock->srcTaskId, pTask->info.selfChildId, checkpointId); // there are still some upstream tasks not send checkpoint request, do nothing and wait for then - int32_t notReady = streamAlignCheckpoint(pTask); - int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); - if (notReady > 0) { - stDebug("s-task:%s received checkpoint block, idx:%d, %d upstream tasks not send checkpoint info yet, total:%d", - id, pTask->info.selfChildId, notReady, num); + if (pActiveInfo->allUpstreamTriggerRecv != 1) { streamFreeQitem((SStreamQueueItem*)pBlock); return code; } + int32_t num = streamTaskGetNumOfUpstream(pTask); if (taskLevel == TASK_LEVEL__SINK) { - stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, send ready msg to upstream", - id, num); + stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, send ready msg to upstream", id, num); streamFreeQitem((SStreamQueueItem*)pBlock); streamTaskBuildCheckpoint(pTask); } else { // source & agg tasks need to forward the checkpoint msg downwards - stDebug("s-task:%s process checkpoint block, all %d upstreams sent checkpoint msgs, continue forwards msg", id, - num); + stDebug("s-task:%s process checkpoint-trigger block, all %d upstreams sent, forwards to downstream", id, num); - // set the needed checked downstream tasks, only when all downstream tasks do checkpoint complete, this task - // can start local checkpoint procedure - pTask->chkInfo.numOfNotReady = streamTaskGetNumOfDownstream(pTask); + flushStateDataInExecutor(pTask, (SStreamQueueItem*)pBlock); - // Put the checkpoint block into inputQ, to make sure all blocks with less version have been handled by this task - // already. And then, dispatch check point msg to all downstream tasks - code = continueDispatchCheckpointBlock(pBlock, pTask); + // Put the checkpoint-trigger block into outputQ, to make sure all blocks with less version have been handled by + // this task already. And then, dispatch check point msg to all downstream tasks + code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); } } @@ -193,135 +328,237 @@ int32_t streamProcessCheckpointBlock(SStreamTask* pTask, SStreamDataBlock* pBloc * All down stream tasks have successfully completed the check point task. * Current stream task is allowed to start to do checkpoint things in ASYNC model. */ -int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask) { +int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, + int32_t downstreamTaskId) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG); + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + + const char* id = pTask->id.idStr; + bool received = false; + int32_t total = streamTaskGetNumOfDownstream(pTask); + ASSERT(total > 0); + + // 1. not in checkpoint status now + SStreamTaskState* pStat = streamTaskGetStatus(pTask); + if (pStat->state != TASK_STATUS__CK) { + stError("s-task:%s status:%s discard checkpoint-ready msg from task:0x%x", id, pStat->name, downstreamTaskId); + return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + } + + // 2. expired checkpoint-ready msg, invalid checkpoint-ready msg + if (pTask->chkInfo.checkpointId > checkpointId || pInfo->activeId != checkpointId) { + stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64 + ") from task:0x%x, expired and discard ", + id, pStat->name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId); + return -1; + } + + taosThreadMutexLock(&pInfo->lock); // only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task - int32_t notReady = atomic_sub_fetch_32(&pTask->chkInfo.numOfNotReady, 1); - ASSERT(notReady >= 0); + int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList); + for (int32_t i = 0; i < size; ++i) { + STaskDownstreamReadyInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i); + if (p->downstreamTaskId == downstreamTaskId) { + received = true; + break; + } + } + + if (received) { + stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, ignore. %d/%d downstream not ready", id, + downstreamTaskId, (int32_t)(total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)), total); + } else { + STaskDownstreamReadyInfo info = {.recvTs = taosGetTimestampMs(), + .downstreamTaskId = downstreamTaskId, + .checkpointId = pInfo->activeId, + .transId = pInfo->transId, + .streamId = pTask->id.streamId, + .downstreamNodeId = downstreamNodeId}; + taosArrayPush(pInfo->pCheckpointReadyRecvList, &info); + } + + int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList); + int32_t transId = pInfo->transId; + taosThreadMutexUnlock(&pInfo->lock); if (notReady == 0) { - stDebug("s-task:%s all downstream tasks have completed the checkpoint, start to do checkpoint for current task", - pTask->id.idStr); - appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT); - } else { - int32_t total = streamTaskGetNumOfDownstream(pTask); - stDebug("s-task:%s %d/%d downstream tasks are not ready, wait", pTask->id.idStr, notReady, total); + stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task", id); + appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId); } return 0; } -void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { - pTask->chkInfo.checkpointingId = 0; // clear the checkpoint id - pTask->chkInfo.failedId = 0; - pTask->chkInfo.startTs = 0; // clear the recorded start time - pTask->chkInfo.numOfNotReady = 0; - pTask->chkInfo.transId = 0; - pTask->chkInfo.dispatchCheckpointTrigger = false; - pTask->chkInfo.downstreamAlignNum = 0; +int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + int64_t now = taosGetTimestampMs(); + int32_t numOfConfirmed = 0; + taosThreadMutexLock(&pInfo->lock); + for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { + STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); + if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { + pReadyInfo->sendCompleted = 1; + stDebug("s-task:%s send checkpoint-ready msg to upstream:0x%x confirmed, checkpointId:%" PRId64 " ts:%" PRId64, + pTask->id.idStr, upstreamTaskId, checkpointId, now); + break; + } + } + + for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { + STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); + if (pReadyInfo->sendCompleted == 1) { + numOfConfirmed += 1; + } + } + + stDebug("s-task:%s send checkpoint-ready msg to %d upstream confirmed, checkpointId:%" PRId64, pTask->id.idStr, + numOfConfirmed, checkpointId); + + taosThreadMutexUnlock(&pInfo->lock); + return TSDB_CODE_SUCCESS; +} + +void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { + pTask->chkInfo.startTs = 0; // clear the recorded start time + + streamTaskClearActiveInfo(pTask->chkInfo.pActiveInfo); streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks if (clearChkpReadyMsg) { streamClearChkptReadyMsg(pTask); } } -int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { - SStreamMeta* pMeta = p->pMeta; +int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq) { + SStreamMeta* pMeta = pTask->pMeta; int32_t vgId = pMeta->vgId; - const char* id = p->id.idStr; int32_t code = 0; - SCheckpointInfo* pCKInfo = &p->chkInfo; + const char* id = pTask->id.idStr; + SCheckpointInfo* pInfo = &pTask->chkInfo; - // fill-history task, rsma task, and sink task will not generate the checkpoint - if ((p->info.fillHistory == 1) || (p->info.taskLevel > TASK_LEVEL__SINK)) { - return code; + taosThreadMutexLock(&pTask->lock); + + if (pReq->checkpointId <= pInfo->checkpointId) { + stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " no need to update the checkpoint info, updated checkpointId:%" PRId64 " checkpointVer:%" PRId64 " ignored", + id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer); + taosThreadMutexUnlock(&pTask->lock); + + { // destroy the related fill-history tasks + // drop task should not in the meta-lock, and drop the related fill-history task now + streamMetaWUnLock(pMeta); + if (pReq->dropRelHTask) { + streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d", + id, vgId, pReq->taskId, numOfTasks); + } + + streamMetaWLock(pMeta); + } + + return TSDB_CODE_SUCCESS; } - taosThreadMutexLock(&p->lock); + SStreamTaskState* pStatus = streamTaskGetStatus(pTask); - SStreamTaskState* pStatus = streamTaskGetStatus(p); - ETaskStatus prevStatus = pStatus->state; + stDebug("s-task:%s vgId:%d status:%s start to update the checkpoint info, checkpointId:%" PRId64 "->%" PRId64 + " checkpointVer:%" PRId64 "->%" PRId64 " checkpointTs:%" PRId64 "->%" PRId64, + id, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId, pInfo->checkpointVer, pReq->checkpointVer, + pInfo->checkpointTime, pReq->checkpointTs); - if (pStatus->state == TASK_STATUS__CK) { - ASSERT(pCKInfo->checkpointId <= pCKInfo->checkpointingId && pCKInfo->checkpointingId == checkpointId && - pCKInfo->checkpointVer <= pCKInfo->processedVer); + if (pStatus->state != TASK_STATUS__DROPPING) { + ASSERT(pInfo->checkpointId <= pReq->checkpointId && pInfo->checkpointVer <= pReq->checkpointVer); - pCKInfo->checkpointId = pCKInfo->checkpointingId; - pCKInfo->checkpointVer = pCKInfo->processedVer; - pCKInfo->checkpointTime = pCKInfo->startTs; + pInfo->checkpointId = pReq->checkpointId; + pInfo->checkpointVer = pReq->checkpointVer; + pInfo->checkpointTime = pReq->checkpointTs; - streamTaskClearCheckInfo(p, false); - taosThreadMutexUnlock(&p->lock); + streamTaskClearCheckInfo(pTask, false); - code = streamTaskHandleEvent(p->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + // todo handle error + if (pStatus->state == TASK_STATUS__CK) { + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); + } else { + stDebug("s-task:0x%x vgId:%d not handle checkpoint-done event, status:%s", pReq->taskId, vgId, pStatus->name); + } } else { - stDebug("s-task:%s vgId:%d status:%s not keep the checkpoint metaInfo, checkpoint:%" PRId64 " failed", id, vgId, - pStatus->name, pCKInfo->checkpointingId); - taosThreadMutexUnlock(&p->lock); + stDebug("s-task:0x%x vgId:%d status:%s not update checkpoint info, checkpointId:%" PRId64 "->%" PRId64 " failed", + pReq->taskId, vgId, pStatus->name, pInfo->checkpointId, pReq->checkpointId); + taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } + if (pReq->dropRelHTask) { + stDebug("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint", + pReq->taskId, vgId, pReq->hTaskId); + CLEAR_RELATED_FILLHISTORY_TASK(pTask); + } + + stDebug("s-task:0x%x set the persistent status attr to be ready, prev:%s, status in sm:%s", pReq->taskId, + streamTaskGetStatusStr(pTask->status.taskStatus), streamTaskGetStatus(pTask)->name); + + pTask->status.taskStatus = TASK_STATUS__READY; + + code = streamMetaSaveTask(pMeta, pTask); if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s vgId:%d handle event:checkpoint-done failed", id, vgId); + stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, + vgId, pReq->checkpointId, terrstr()); return code; } - stDebug("vgId:%d s-task:%s level:%d open upstream inputQ, save status after checkpoint, checkpointId:%" PRId64 - ", Ver(saved):%" PRId64 " currentVer:%" PRId64 ", status: ready, prev:%s", - vgId, id, p->info.taskLevel, checkpointId, pCKInfo->checkpointVer, pCKInfo->nextProcessVer, - streamTaskGetStatusStr(prevStatus)); + taosThreadMutexUnlock(&pTask->lock); + streamMetaWUnLock(pMeta); - // save the task if not sink task - if (p->info.taskLevel <= TASK_LEVEL__SINK) { - streamMetaWLock(pMeta); - - code = streamMetaSaveTask(pMeta, p); - if (code != TSDB_CODE_SUCCESS) { - streamMetaWUnLock(pMeta); - stError("s-task:%s vgId:%d failed to save task info after do checkpoint, checkpointId:%" PRId64 ", since %s", id, - vgId, checkpointId, terrstr()); - return code; - } - - code = streamMetaCommit(pMeta); - if (code != TSDB_CODE_SUCCESS) { - stError("s-task:%s vgId:%d failed to commit stream meta after do checkpoint, checkpointId:%" PRId64 ", since %s", - id, vgId, checkpointId, terrstr()); - } - - streamMetaWUnLock(pMeta); + // drop task should not in the meta-lock, and drop the related fill-history task now + if (pReq->dropRelHTask) { + streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, (int32_t) pReq->hTaskId, numOfTasks); } - return code; + streamMetaWLock(pMeta); + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + + return TSDB_CODE_SUCCESS; } void streamTaskSetFailedCheckpointId(SStreamTask* pTask) { - pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; + pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId; stDebug("s-task:%s mark the checkpointId:%" PRId64 " (transId:%d) failed", pTask->id.idStr, - pTask->chkInfo.checkpointingId, pTask->chkInfo.transId); + pTask->chkInfo.pActiveInfo->activeId, pTask->chkInfo.pActiveInfo->transId); } static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* list) { + char buf[128] = {0}; + char* file = taosMemoryCalloc(1, strlen(path) + 32); sprintf(file, "%s%s%s", path, TD_DIRSEP, "META_TMP"); int32_t code = downloadCheckpointDataByName(id, "META", file); if (code != 0) { - stDebug("chkp failed to download meta file:%s", file); + stDebug("%s chkp failed to download meta file:%s", id, file); taosMemoryFree(file); return code; } TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); - char buf[128] = {0}; + if (pFile == NULL) { + stError("%s failed to open meta file:%s for checkpoint", id, file); + code = -1; + return code; + } + if (taosReadFile(pFile, buf, sizeof(buf)) <= 0) { - stError("chkp failed to read meta file:%s", file); + stError("%s failed to read meta file:%s for checkpoint", id, file); code = -1; } else { - int32_t len = strlen(buf); + int32_t len = strnlen(buf, tListLen(buf)); for (int i = 0; i < len; i++) { if (buf[i] == '\n') { char* item = taosMemoryCalloc(1, i + 1); @@ -334,6 +571,7 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } } } + taosCloseFile(&pFile); taosRemoveFile(file); taosMemoryFree(file); @@ -341,58 +579,75 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } int32_t uploadCheckpointData(void* param) { - SAsyncUploadArg* arg = param; + SAsyncUploadArg* pParam = param; char* path = NULL; int32_t code = 0; - SArray* toDelFiles = taosArrayInit(4, sizeof(void*)); - char* taskStr = arg->taskId ? arg->taskId : "NULL"; + SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); + char* taskStr = pParam->taskId ? pParam->taskId : "NULL"; - void* pBackend = taskAcquireDb(arg->dbRefId); + void* pBackend = taskAcquireDb(pParam->dbRefId); if (pBackend == NULL) { stError("s-task:%s failed to acquire db", taskStr); - taosMemoryFree(arg->taskId); - taosMemoryFree(arg); + taosMemoryFree(pParam->taskId); + taosMemoryFree(pParam); return -1; } - if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, ((SStreamMeta*)arg->pMeta)->bkdChkptMgt, arg->chkpId, - (int8_t)(arg->type), &path, toDelFiles)) != 0) { - stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", taskStr, arg->chkpId); + if ((code = taskDbGenChkpUploadData(pParam->pTask->pBackend, ((SStreamMeta*)pParam->pMeta)->bkdChkptMgt, + pParam->chkpId, (int8_t)(pParam->type), &path, toDelFiles)) != 0) { + stError("s-task:%s failed to gen upload checkpoint:%" PRId64, taskStr, pParam->chkpId); } - if (arg->type == DATA_UPLOAD_S3) { - if (code == 0 && (code = getCheckpointDataMeta(arg->taskId, path, toDelFiles)) != 0) { - stError("s-task:%s failed to get checkpointId:%" PRId64 " meta", taskStr, arg->chkpId); + if (pParam->type == DATA_UPLOAD_S3) { + if (code == 0 && (code = getCheckpointDataMeta(pParam->taskId, path, toDelFiles)) != 0) { + stError("s-task:%s failed to get checkpointData for checkpointId:%" PRId64 " meta", taskStr, pParam->chkpId); } } - if (code == 0 && (code = streamTaskBackupCheckpoint(arg->taskId, path)) != 0) { - stError("s-task:%s failed to upload checkpointId:%" PRId64, taskStr, arg->chkpId); + if (code == TSDB_CODE_SUCCESS) { + code = streamTaskUploadCheckpoint(pParam->taskId, path); + if (code == TSDB_CODE_SUCCESS) { + stDebug("s-task:%s upload checkpointId:%" PRId64 " to remote succ", taskStr, pParam->chkpId); + } else { + stError("s-task:%s failed to upload checkpointId:%" PRId64 " data:%s", taskStr, pParam->chkpId, path); + } } - taskReleaseDb(arg->dbRefId); + taskReleaseDb(pParam->dbRefId); if (code == 0) { - for (int i = 0; i < taosArrayGetSize(toDelFiles); i++) { - char* p = taosArrayGetP(toDelFiles, i); - code = deleteCheckpointFile(arg->taskId, p); - stDebug("s-task:%s try to del file: %s", taskStr, p); + int32_t size = taosArrayGetSize(toDelFiles); + stDebug("s-task:%s remove redundant %d files", taskStr, size); + + for (int i = 0; i < size; i++) { + char* pName = taosArrayGetP(toDelFiles, i); + code = deleteCheckpointFile(pParam->taskId, pName); if (code != 0) { + stDebug("s-task:%s failed to del file: %s", taskStr, pName); break; } } + + stDebug("s-task:%s remove redundant files done", taskStr); } taosArrayDestroyP(toDelFiles, taosMemoryFree); - taosRemoveDir(path); + + if (code == TSDB_CODE_SUCCESS) { + stDebug("s-task:%s remove local checkpointId:%" PRId64 " data %s", taskStr, pParam->chkpId, path); + taosRemoveDir(path); + } else { + stDebug("s-task:%s update checkpointId:%" PRId64 " keep local checkpoint data", taskStr, pParam->chkpId); + } + taosMemoryFree(path); - taosMemoryFree(arg->taskId); - taosMemoryFree(arg); + taosMemoryFree(pParam->taskId); + taosMemoryFree(pParam); return code; } -int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t chkpId, char* taskId) { +int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointId, char* taskId) { ECHECKPOINT_BACKUP_TYPE type = streamGetCheckpointBackupType(); if (type == DATA_UPLOAD_DISABLE) { return 0; @@ -405,7 +660,7 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t chkpId, cha SAsyncUploadArg* arg = taosMemoryCalloc(1, sizeof(SAsyncUploadArg)); arg->type = type; arg->taskId = taosStrdup(taskId); - arg->chkpId = chkpId; + arg->chkpId = checkpointId; arg->pTask = pTask; arg->dbRefId = taskGetDBRef(pTask->pBackend); arg->pMeta = pTask->pMeta; @@ -416,7 +671,7 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t chkpId, cha int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { int32_t code = TSDB_CODE_SUCCESS; int64_t startTs = pTask->chkInfo.startTs; - int64_t ckId = pTask->chkInfo.checkpointingId; + int64_t ckId = pTask->chkInfo.pActiveInfo->activeId; const char* id = pTask->id.idStr; bool dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT); SStreamMeta* pMeta = pTask->pMeta; @@ -446,9 +701,11 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { } } - // clear the checkpoint info, and commit the newest checkpoint info if all works are done successfully - if (code == TSDB_CODE_SUCCESS) { - code = streamSaveTaskCheckpointInfo(pTask, ckId); + // update the latest checkpoint info if all works are done successfully, for rsma, the pMsgCb is null. + if (code == TSDB_CODE_SUCCESS && (pTask->pMsgCb != NULL)) { + STaskId* pHTaskId = &pTask->hTaskInfo.id; + code = streamBuildAndSendCheckpointUpdateMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, pHTaskId, &pTask->chkInfo, + dropRelHTask); if (code == TSDB_CODE_SUCCESS) { code = streamTaskRemoteBackupCheckpoint(pTask, ckId, (char*)id); if (code != TSDB_CODE_SUCCESS) { @@ -459,22 +716,6 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { } } - if ((code == TSDB_CODE_SUCCESS) && dropRelHTask) { - // transferred from the halt status, it is done the fill-history procedure and finish with the checkpoint - // free it and remove fill-history task from disk meta-store - taosThreadMutexLock(&pTask->lock); - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - SStreamTaskId hTaskId = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; - - stDebug("s-task:%s fill-history finish checkpoint done, drop related fill-history task:0x%x", id, hTaskId.taskId); - streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &hTaskId, 1); - } else { - stWarn("s-task:%s related fill-history task:0x%x is erased", id, (int32_t)pTask->hTaskInfo.id.taskId); - } - - taosThreadMutexUnlock(&pTask->lock); - } - // clear the checkpoint info if failed if (code != TSDB_CODE_SUCCESS) { taosThreadMutexLock(&pTask->lock); @@ -488,12 +729,260 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { double el = (taosGetTimestampMs() - startTs) / 1000.0; stInfo("s-task:%s vgId:%d level:%d, checkpointId:%" PRId64 " ver:%" PRId64 " elapsed time:%.2f Sec, %s ", id, - pTask->pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el, + pMeta->vgId, pTask->info.taskLevel, ckId, pTask->chkInfo.checkpointVer, el, (code == TSDB_CODE_SUCCESS) ? "succ" : "failed"); return code; } +void checkpointTriggerMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + if (++pActiveInfo->checkCounter < 100) { + taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + return; + } + + pActiveInfo->checkCounter = 0; + stDebug("s-task:%s vgId:%d checkpoint-trigger monitor in tmr, ts:%" PRId64, pTask->id.idStr, vgId, now); + + taosThreadMutexLock(&pTask->lock); + SStreamTaskState* pState = streamTaskGetStatus(pTask); + if (pState->state != TASK_STATUS__CK) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s vgId:%d not in checkpoint status, quit from monitor checkpoint-trigger, ref:%d", pTask->id.idStr, + vgId, ref); + + taosThreadMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + taosThreadMutexUnlock(&pTask->lock); + + taosThreadMutexLock(&pActiveInfo->lock); + + // send msg to retrieve checkpoint trigger msg + SArray* pList = pTask->upstreamInfo.pList; + ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE); + SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); + + for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i); + + bool recved = false; + for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) { + STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j); + if (pInfo->nodeId == pReady->upstreamNodeId) { + recved = true; + break; + } + } + + if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message + streamTaskOpenUpstreamInput(pTask, pInfo->taskId); + taosArrayPush(pNotSendList, pInfo); + } + } + + // do send retrieve checkpoint trigger msg to upstream + int32_t size = taosArrayGetSize(pNotSendList); + doSendRetrieveTriggerMsg(pTask, pNotSendList); + taosThreadMutexUnlock(&pActiveInfo->lock); + + // check every 100ms + if (size > 0) { + stDebug("s-task:%s start to monitor checkpoint-trigger in 10s", id); + taosTmrReset(checkpointTriggerMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pChkptTriggerTmr); + } else { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s all checkpoint-trigger recved, quit from monitor checkpoint-trigger tmr, ref:%d", id, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + } + + taosArrayDestroy(pNotSendList); +} + +int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList) { + int32_t code = 0; + int32_t vgId = pTask->pMeta->vgId; + const char* pId = pTask->id.idStr; + int32_t size = taosArrayGetSize(pNotSendList); + int32_t numOfUpstream = streamTaskGetNumOfUpstream(pTask); + + if (size <= 0) { + stDebug("s-task:%s all upstream checkpoint trigger recved, no need to send retrieve", pId); + return code; + } + + stDebug("s-task:%s %d/%d not recv checkpoint-trigger from upstream(s), start to send trigger-retrieve", pId, size, + numOfUpstream); + + for (int32_t i = 0; i < size; i++) { + SStreamUpstreamEpInfo* pUpstreamTask = taosArrayGet(pNotSendList, i); + + SRetrieveChkptTriggerReq* pReq = rpcMallocCont(sizeof(SRetrieveChkptTriggerReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + stError("vgId:%d failed to create msg to retrieve trigger msg for task:%s exec, code:out of memory", vgId, pId); + continue; + } + + pReq->head.vgId = htonl(pUpstreamTask->nodeId); + pReq->streamId = pTask->id.streamId; + pReq->downstreamTaskId = pTask->id.taskId; + pReq->downstreamNodeId = vgId; + pReq->upstreamTaskId = pUpstreamTask->taskId; + pReq->upstreamNodeId = pUpstreamTask->nodeId; + pReq->checkpointId = pTask->chkInfo.pActiveInfo->activeId; + + + SRpcMsg rpcMsg = {0}; + initRpcMsg(&rpcMsg, TDMT_STREAM_RETRIEVE_TRIGGER, pReq, sizeof(SRetrieveChkptTriggerReq)); + + code = tmsgSendReq(&pUpstreamTask->epSet, &rpcMsg); + stDebug("s-task:%s vgId:%d send checkpoint-trigger retrieve msg to 0x%x(vgId:%d) checkpointId:%" PRId64, pId, vgId, + pUpstreamTask->taskId, pUpstreamTask->nodeId, pReq->checkpointId); + } + + return TSDB_CODE_SUCCESS; +} + +bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) { + int64_t now = taosGetTimestampMs(); + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + SStreamTaskState* pStatus = streamTaskGetStatus(pTask); + + if (pStatus->state != TASK_STATUS__CK) { + return false; + } + + taosThreadMutexLock(&pInfo->lock); + if (!pInfo->dispatchTrigger) { + taosThreadMutexUnlock(&pInfo->lock); + return false; + } + + for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (pSendInfo->nodeId != downstreamNodeId) { + continue; + } + + // has send trigger msg to downstream node, + double before = (now - pSendInfo->sendTs) / 1000.0; + if (pSendInfo->recved) { + stWarn("s-task:%s checkpoint-trigger msg already send at:%" PRId64 + "(%.2fs before) and recv confirmed by downstream:0x%x, checkpointId:%" PRId64 ", transId:%d", + id, pSendInfo->sendTs, before, pSendInfo->taskId, pInfo->activeId, pInfo->transId); + } else { + stWarn("s-task:%s checkpoint-trigger already send at:%" PRId64 "(%.2fs before), checkpointId:%" PRId64 + ", transId:%d", + id, pSendInfo->sendTs, before, pInfo->activeId, pInfo->transId); + } + + taosThreadMutexUnlock(&pInfo->lock); + return true; + } + + ASSERT(0); + return false; +} + +void streamTaskGetTriggerRecvStatus(SStreamTask* pTask, int32_t* pRecved, int32_t* pTotal) { + *pRecved = taosArrayGetSize(pTask->chkInfo.pActiveInfo->pReadyMsgList); + + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + *pTotal = 1; + } else { + *pTotal = streamTaskGetNumOfUpstream(pTask); + } +} + +// record the dispatch checkpoint trigger info in the list +void streamTaskInitTriggerDispatchInfo(SStreamTask* pTask) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + + int64_t now = taosGetTimestampMs(); + taosThreadMutexLock(&pInfo->lock); + + // outputQ should be empty here + ASSERT(streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue) == 0); + + pInfo->dispatchTrigger = true; + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + STaskDispatcherFixed* pDispatch = &pTask->outputInfo.fixedDispatcher; + + STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pDispatch->nodeId, .taskId = pDispatch->taskId}; + taosArrayPush(pInfo->pDispatchTriggerList, &p); + } else { + for (int32_t i = 0; i < streamTaskGetNumOfDownstream(pTask); ++i) { + SVgroupInfo* pVgInfo = taosArrayGet(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos, i); + + STaskTriggerSendInfo p = {.sendTs = now, .recved = false, .nodeId = pVgInfo->vgId, .taskId = pVgInfo->taskId}; + taosArrayPush(pInfo->pDispatchTriggerList, &p); + } + } + + taosThreadMutexUnlock(&pInfo->lock); +} + +int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + + int32_t num = 0; + taosThreadMutexLock(&pInfo->lock); + for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (p->recved) { + num ++; + } + } + taosThreadMutexUnlock(&pInfo->lock); + return num; +} + +void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) { + SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; + + int32_t taskId = 0; + taosThreadMutexLock(&pInfo->lock); + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); + if (p->nodeId == vgId) { + ASSERT(p->recved == false); + + p->recved = true; + p->recvTs = taosGetTimestampMs(); + taskId = p->taskId; + break; + } + } + + taosThreadMutexUnlock(&pInfo->lock); + + int32_t numOfConfirmed = streamTaskGetNumOfConfirmed(pTask); + int32_t total = streamTaskGetNumOfDownstream(pTask); + stDebug("s-task:%s set downstream:0x%x(vgId:%d) checkpoint-trigger dispatch confirmed, total confirmed:%d/%d", + pTask->id.idStr, taskId, vgId, numOfConfirmed, total); + + ASSERT(taskId != 0); +} + static int32_t uploadCheckpointToS3(const char* id, const char* path) { TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) return -1; @@ -553,14 +1042,14 @@ ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType() { } } -int32_t streamTaskBackupCheckpoint(const char* id, const char* path) { +int32_t streamTaskUploadCheckpoint(const char* id, const char* path) { if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { - stError("streamTaskBackupCheckpoint parameters invalid"); + stError("invalid parameters in upload checkpoint, %s", id); return -1; } if (strlen(tsSnodeAddress) != 0) { - return uploadRsync(id, path); + return uploadByRsync(id, path); } else if (tsS3StreamEnabled) { return uploadCheckpointToS3(id, path); } @@ -571,7 +1060,7 @@ int32_t streamTaskBackupCheckpoint(const char* id, const char* path) { // fileName: CURRENT int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName) { if (id == NULL || fname == NULL || strlen(id) == 0 || strlen(fname) == 0 || strlen(fname) >= PATH_MAX) { - stError("uploadCheckpointByName parameters invalid"); + stError("down load checkpoint data parameters invalid"); return -1; } @@ -584,9 +1073,9 @@ int32_t downloadCheckpointDataByName(const char* id, const char* fname, const ch return 0; } -int32_t streamTaskDownloadCheckpointData(char* id, char* path) { +int32_t streamTaskDownloadCheckpointData(const char* id, char* path) { if (id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX) { - stError("streamTaskDownloadCheckpointData parameters invalid"); + stError("down checkpoint data parameters invalid"); return -1; } @@ -615,6 +1104,7 @@ int32_t deleteCheckpoint(const char* id) { int32_t deleteCheckpointFile(const char* id, const char* name) { char object[128] = {0}; snprintf(object, sizeof(object), "%s/%s", id, name); + char* tmp = object; s3DeleteObjects((const char**)&tmp, 1); return 0; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f9bd0bbfc0..9cdc668ec4 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -23,12 +23,6 @@ typedef struct SBlockName { char parTbName[TSDB_TABLE_NAME_LEN]; } SBlockName; -typedef struct { - int32_t upStreamTaskId; - SEpSet upstreamNodeEpset; - SRpcMsg msg; -} SStreamChkptReadyInfo; - static void doRetryDispatchData(void* param, void* tmrId); static int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet); static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq); @@ -85,12 +79,14 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r void* buf = NULL; int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList); ASSERT(sz > 0); + for (int32_t i = 0; i < sz; i++) { req->reqId = tGenIdPI64(); - SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); req->dstNodeId = pEpInfo->nodeId; req->dstTaskId = pEpInfo->taskId; int32_t len; + tEncodeSize(tEncodeStreamRetrieveReq, req, len, code); if (code != 0) { ASSERT(0); @@ -115,7 +111,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r code = tmsgSendReq(&pEpInfo->epSet, &rpcMsg); if (code != 0) { - ASSERT(0); rpcFreeCont(buf); return code; } @@ -124,6 +119,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); } + return code; } @@ -257,6 +253,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; code = tInitStreamDispatchReq(pReq, pTask, pData->srcVgId, numOfBlocks, downstreamTaskId, pData->type); if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); return code; } @@ -348,7 +345,7 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; - stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), id:%d", id, + stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d", id, pTask->info.selfChildId, 1, downstreamTaskId, vgId, msgId); code = doSendDispatchMsg(pTask, pDispatchMsg, vgId, pEpSet); @@ -364,8 +361,8 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch for (int32_t i = 0; i < numOfVgroups; i++) { if (pDispatchMsg[i].blockNum > 0) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, - pTask->info.selfChildId, pDispatchMsg[i].blockNum, pVgInfo->vgId); + stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", id, pTask->info.selfChildId, + pDispatchMsg[i].blockNum, pVgInfo->vgId); code = doSendDispatchMsg(pTask, &pDispatchMsg[i], pVgInfo->vgId, &pVgInfo->epSet); if (code < 0) { @@ -379,8 +376,7 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch } } - stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfVgroups, - msgId); + stDebug("s-task:%s complete shuffle-dispatch blocks to all %d vnodes, msgId:%d", id, numOfVgroups, msgId); } return code; @@ -432,14 +428,14 @@ static void doRetryDispatchData(void* param, void* tmrId) { } } - stDebug("s-task:%s complete re-try shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, + stDebug("s-task:%s complete retry shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, numOfFailed, msgId); } else { int32_t vgId = pTask->outputInfo.fixedDispatcher.nodeId; SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; - stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), id:%d", id, + stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d", id, pTask->info.selfChildId, 1, downstreamTaskId, vgId, msgId); code = doSendDispatchMsg(pTask, pReq, vgId, pEpSet); @@ -470,13 +466,13 @@ static void doRetryDispatchData(void* param, void* tmrId) { void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) { pTask->msgInfo.retryCount++; - stWarn("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr, + stTrace("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr, waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount); - if (pTask->msgInfo.pTimer != NULL) { - taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pTimer); + if (pTask->msgInfo.pRetryTmr != NULL) { + taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); } else { - pTask->msgInfo.pTimer = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer); + pTask->msgInfo.pRetryTmr = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer); } } @@ -501,7 +497,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S if (pDataBlock->info.parTbName[0]) { if(pTask->subtableWithoutMd5 != 1 && !isAutoTableName(pDataBlock->info.parTbName) && - !alreadyAddGroupId(pDataBlock->info.parTbName) && + !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && groupId != 0){ if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId); @@ -556,9 +552,10 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { const char* id = pTask->id.idStr; int32_t numOfElems = streamQueueGetNumOfItems(pTask->outputq.queue); if (numOfElems > 0) { - double size = SIZE_IN_MiB(taosQueueMemorySize(pTask->outputq.queue->pQueue)); - stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d, size:%.2fMiB", id, - numOfElems, size); + double size = SIZE_IN_MiB(taosQueueMemorySize(pTask->outputq.queue->pQueue)); + int32_t numOfUnAccessed = streamQueueGetNumOfUnAccessedItems(pTask->outputq.queue); + stDebug("s-task:%s start to dispatch intermediate block to downstream, elem in outputQ:%d/%d, size:%.2fMiB", id, + numOfUnAccessed, numOfElems, size); } // to make sure only one dispatch is running @@ -569,8 +566,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } - if (pTask->chkInfo.dispatchCheckpointTrigger) { - stDebug("s-task:%s already send checkpoint trigger, not dispatch anymore", id); + if (pTask->chkInfo.pActiveInfo->dispatchTrigger) { + stDebug("s-task:%s already send checkpoint-trigger, no longer dispatch any other data", id); atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); return 0; } @@ -585,8 +582,9 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return 0; } - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK || pBlock->type == STREAM_INPUT__CHECKPOINT_TRIGGER || - pBlock->type == STREAM_INPUT__TRANS_STATE); + int32_t type = pBlock->type; + ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT_TRIGGER || + type == STREAM_INPUT__TRANS_STATE); pTask->execInfo.dispatch += 1; pTask->msgInfo.startTs = taosGetTimestampMs(); @@ -597,6 +595,10 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { } else { // todo handle build dispatch msg failed } + if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { + streamTaskInitTriggerDispatchInfo(pTask); + } + int32_t retryCount = 0; while (1) { code = sendDispatchMsg(pTask, pTask->msgInfo.pData); @@ -629,43 +631,185 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } -// this function is usually invoked by sink/agg task -int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { - int32_t num = taosArrayGetSize(pTask->pReadyMsgList); +int32_t initCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamNodeId, int32_t upstreamTaskId, int32_t childId, + int64_t checkpointId, SRpcMsg* pMsg) { + int32_t code = 0; + int32_t tlen = 0; + void* buf = NULL; + + SStreamCheckpointReadyMsg req = {0}; + req.downstreamNodeId = pTask->pMeta->vgId; + req.downstreamTaskId = pTask->id.taskId; + req.streamId = pTask->id.streamId; + req.checkpointId = checkpointId; + req.childId = childId; + req.upstreamNodeId = upstreamNodeId; + req.upstreamTaskId = upstreamTaskId; + + tEncodeSize(tEncodeStreamCheckpointReadyMsg, &req, tlen, code); + if (code < 0) { + return -1; + } + + buf = rpcMallocCont(sizeof(SMsgHead) + tlen); + if (buf == NULL) { + return -1; + } + + ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) { + rpcFreeCont(buf); + return code; + } + tEncoderClear(&encoder); + + initRpcMsg(pMsg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); + return TSDB_CODE_SUCCESS; +} + +static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { + SStreamTask* pTask = param; + int32_t vgId = pTask->pMeta->vgId; + const char* id = pTask->id.idStr; + + // check the status every 100ms + if (streamTaskShouldStop(pTask)) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s vgId:%d quit from monitor checkpoint-trigger, ref:%d", id, vgId, ref); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + if (++pActiveInfo->sendReadyCheckCounter < 100) { + taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); + return; + } + + pActiveInfo->sendReadyCheckCounter = 0; + stDebug("s-task:%s in sending checkpoint-ready msg monitor timer", id); + + taosThreadMutexLock(&pActiveInfo->lock); + + SArray* pList = pActiveInfo->pReadyMsgList; + SArray* pNotRspList = taosArrayInit(4, sizeof(int32_t)); + + int32_t num = taosArrayGetSize(pList); ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); for (int32_t i = 0; i < num; ++i) { - SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, i); - tmsgSendReq(&pInfo->upstreamNodeEpset, &pInfo->msg); + STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); + if (pInfo->sendCompleted == 1) { + continue; + } - stDebug("s-task:%s level:%d checkpoint ready msg sent to upstream:0x%x", pTask->id.idStr, pTask->info.taskLevel, - pInfo->upStreamTaskId); + taosArrayPush(pNotRspList, &pInfo->upstreamTaskId); + stDebug("s-task:%s vgId:%d level:%d checkpoint-ready rsp from upstream:0x%x not confirmed yet", id, vgId, + pTask->info.taskLevel, pInfo->upstreamTaskId); } - taosArrayClear(pTask->pReadyMsgList); - stDebug("s-task:%s level:%d checkpoint ready msg sent to all %d upstreams", pTask->id.idStr, pTask->info.taskLevel, - num); + int32_t checkpointId = pActiveInfo->activeId; + + int32_t notRsp = taosArrayGetSize(pNotRspList); + if (notRsp > 0) { // send checkpoint-ready msg again + for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { + int32_t taskId = *(int32_t*)taosArrayGet(pNotRspList, i); + + for (int32_t j = 0; j < num; ++j) { + STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j); + if (taskId == pReadyInfo->upstreamTaskId) { // send msg again + + SRpcMsg msg = {0}; + initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, + checkpointId, &msg); + tmsgSendReq(&pReadyInfo->upstreamNodeEpset, &msg); + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x again", id, pTask->info.taskLevel, + pReadyInfo->upstreamTaskId); + } + } + } + + taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); + taosThreadMutexUnlock(&pActiveInfo->lock); + } else { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug( + "s-task:%s vgId:%d recv of checkpoint-ready msg confirmed by all upstream task(s), clear checkpoint-ready msg " + "and quit from timer, ref:%d", + id, vgId, ref); + + streamClearChkptReadyMsg(pTask); + taosThreadMutexUnlock(&pActiveInfo->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + } + + taosArrayDestroy(pNotRspList); +} + +// this function is usually invoked by sink/agg task +int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + const char* id = pTask->id.idStr; + SArray* pList = pActiveInfo->pReadyMsgList; + + taosThreadMutexLock(&pActiveInfo->lock); + + int32_t num = taosArrayGetSize(pList); + ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) == num); + + for (int32_t i = 0; i < num; ++i) { + STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); + + SRpcMsg msg = {0}; + initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, &msg); + tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); + + stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, + pInfo->upstreamTaskId); + } + + taosThreadMutexUnlock(&pActiveInfo->lock); + stDebug("s-task:%s level:%d checkpoint-ready msg sent to all %d upstreams", id, pTask->info.taskLevel, num); + + // start to check if checkpoint ready msg has successfully received by upstream tasks. + if (pTask->info.taskLevel == TASK_LEVEL__SINK || pTask->info.taskLevel == TASK_LEVEL__AGG) { + int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s start checkpoint-ready monitor in 10s, ref:%d ", pTask->id.idStr, ref); + streamMetaAcquireOneTask(pTask); + + if (pActiveInfo->pSendReadyMsgTmr == NULL) { + pActiveInfo->pSendReadyMsgTmr = taosTmrStart(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer); + } else { + taosTmrReset(checkpointReadyMsgSendMonitorFn, 100, pTask, streamTimer, &pActiveInfo->pSendReadyMsgTmr); + } + } return TSDB_CODE_SUCCESS; } // this function is only invoked by source task, and send rsp to mnode int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { - taosThreadMutexLock(&pTask->lock); + SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList; + taosThreadMutexLock(&pTask->chkInfo.pActiveInfo->lock); ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); - if (taosArrayGetSize(pTask->pReadyMsgList) == 1) { - SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, 0); + if (taosArrayGetSize(pList) == 1) { + STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, 0); tmsgSendRsp(&pInfo->msg); - taosArrayClear(pTask->pReadyMsgList); + taosArrayClear(pList); stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); } else { stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel); } - taosThreadMutexUnlock(&pTask->lock); + taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); return TSDB_CODE_SUCCESS; } @@ -790,103 +934,113 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp } int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask) { - SStreamChkptReadyInfo info = {0}; + STaskCheckpointReadyInfo info = { + .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; + streamTaskBuildCheckpointSourceRsp(pReq, pRpcInfo, &info.msg, TSDB_CODE_SUCCESS); - if (pTask->pReadyMsgList == NULL) { - pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo)); + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + taosThreadMutexLock(&pActiveInfo->lock); + + int32_t size = taosArrayGetSize(pActiveInfo->pReadyMsgList); + if (size > 0) { + ASSERT(size == 1); + + STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, 0); + if (pReady->transId == pReq->transId) { + stWarn("s-task:%s repeatly recv checkpoint source msg from mnode, checkpointId:%" PRId64 ", ignore", + pTask->id.idStr, pReq->checkpointId); + } else { + stError("s-task:%s checkpointId:%" PRId64 " transId:%d not completed, new transId:%d checkpointId:%" PRId64 + " recv from mnode", + pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); + ASSERT(0); // failed to handle it + } + } else { + taosArrayPush(pActiveInfo->pReadyMsgList, &info); + stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size + 1); } - taosArrayPush(pTask->pReadyMsgList, &info); + taosThreadMutexUnlock(&pActiveInfo->lock); + return TSDB_CODE_SUCCESS; +} + +int32_t initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, + int32_t childId, SEpSet* pEpset, int64_t checkpointId) { + ASSERT(upstreamTaskId != 0); + + pReadyInfo->upstreamTaskId = upstreamTaskId; + pReadyInfo->upstreamNodeEpset = *pEpset; + pReadyInfo->upstreamNodeId = upstreamNodeId; + pReadyInfo->recvTs = taosGetTimestampMs(); + pReadyInfo->checkpointId = checkpointId; + pReadyInfo->childId = childId; - int32_t size = taosArrayGetSize(pTask->pReadyMsgList); - stDebug("s-task:%s add checkpoint source rsp msg, total:%d", pTask->id.idStr, size); return TSDB_CODE_SUCCESS; } int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, int32_t index, int64_t checkpointId) { - int32_t code = 0; - int32_t tlen = 0; - void* buf = NULL; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { return TSDB_CODE_SUCCESS; } - SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); - SStreamCheckpointReadyMsg req = {0}; - req.downstreamNodeId = pTask->pMeta->vgId; - req.downstreamTaskId = pTask->id.taskId; - req.streamId = pTask->id.streamId; - req.checkpointId = checkpointId; - req.childId = pInfo->childId; - req.upstreamNodeId = pInfo->nodeId; - req.upstreamTaskId = pInfo->taskId; + STaskCheckpointReadyInfo info = {0}; + initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); - tEncodeSize(tEncodeStreamCheckpointReadyMsg, &req, tlen, code); - if (code < 0) { - return -1; + stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 + "-0x%x (vgId:%d) idx:%d", + pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); + + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + + taosThreadMutexLock(&pActiveInfo->lock); + taosArrayPush(pActiveInfo->pReadyMsgList, &info); + + int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); + int32_t total = streamTaskGetNumOfUpstream(pTask); + if (numOfRecv == total) { + stDebug("s-task:%s recv checkpoint-trigger from all upstream, continue", pTask->id.idStr); + pActiveInfo->allUpstreamTriggerRecv = 1; + } else { + ASSERT(numOfRecv <= total); + stDebug("s-task:%s %d/%d checkpoint-trigger recv", pTask->id.idStr, numOfRecv, total); } - buf = rpcMallocCont(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - return -1; - } - - ((SMsgHead*)buf)->vgId = htonl(req.upstreamNodeId); - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, tlen); - if ((code = tEncodeStreamCheckpointReadyMsg(&encoder, &req)) < 0) { - rpcFreeCont(buf); - return code; - } - tEncoderClear(&encoder); - - ASSERT(req.upstreamTaskId != 0); - - SStreamChkptReadyInfo info = {.upStreamTaskId = pInfo->taskId, .upstreamNodeEpset = pInfo->epSet}; - initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); - - stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 - ":0x%x (vgId:%d) idx:%d, vgId:%d", - pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index, - req.upstreamNodeId); - - if (pTask->pReadyMsgList == NULL) { - pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo)); - } - - taosArrayPush(pTask->pReadyMsgList, &info); + taosThreadMutexUnlock(&pActiveInfo->lock); return 0; } void streamClearChkptReadyMsg(SStreamTask* pTask) { - if (pTask->pReadyMsgList == NULL) { + SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; + if (pActiveInfo == NULL) { return; } - for (int i = 0; i < taosArrayGetSize(pTask->pReadyMsgList); i++) { - SStreamChkptReadyInfo* pInfo = taosArrayGet(pTask->pReadyMsgList, i); + for (int i = 0; i < taosArrayGetSize(pActiveInfo->pReadyMsgList); i++) { + STaskCheckpointReadyInfo* pInfo = taosArrayGet(pActiveInfo->pReadyMsgList, i); rpcFreeCont(pInfo->msg.pCont); } - taosArrayClear(pTask->pReadyMsgList); + + taosArrayClear(pActiveInfo->pReadyMsgList); } // this message has been sent successfully, let's try next one. -static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { +static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId, int32_t downstreamNodeId) { stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); if (delayDispatch) { taosThreadMutexLock(&pTask->lock); // we only set the dispatch msg info for current checkpoint trans - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && pTask->chkInfo.checkpointingId == pTask->msgInfo.checkpointId) { - ASSERT(pTask->chkInfo.transId == pTask->msgInfo.transId); - pTask->chkInfo.dispatchCheckpointTrigger = true; - stDebug("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 " transId:%d confirmed", - pTask->id.idStr, pTask->msgInfo.checkpointId, pTask->msgInfo.transId); + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && + pTask->chkInfo.pActiveInfo->activeId == pTask->msgInfo.checkpointId) { + ASSERT(pTask->chkInfo.pActiveInfo->transId == pTask->msgInfo.transId); + stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed", + pTask->id.idStr, downstreamId, pTask->msgInfo.checkpointId, pTask->msgInfo.transId); + + streamTaskSetTriggerDispatchConfirmed(pTask, downstreamNodeId); } else { stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 " transId:%d discard, since expired", pTask->id.idStr, pTask->msgInfo.checkpointId, pTask->msgInfo.transId); @@ -971,7 +1125,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId); taosThreadMutexUnlock(&pTask->lock); - stWarn("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id, + stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, DISPATCH_RETRY_INTERVAL_MS); } else if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) { // todo handle the agg task failure, add test case @@ -979,10 +1133,10 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i pTask->info.taskLevel == TASK_LEVEL__SOURCE) { stError("s-task:%s failed to dispatch checkpoint-trigger msg, checkpointId:%" PRId64 ", set the current checkpoint failed, and send rsp to mnode", - id, pTask->chkInfo.checkpointingId); + id, pTask->chkInfo.pActiveInfo->activeId); { // send checkpoint failure msg to mnode directly - pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; // record the latest failed checkpoint id - pTask->chkInfo.checkpointingId = pTask->chkInfo.checkpointingId; + pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId; // record the latest failed checkpoint id + pTask->chkInfo.pActiveInfo->activeId = pTask->chkInfo.pActiveInfo->activeId; streamTaskSendCheckpointSourceRsp(pTask); } } else { @@ -1036,7 +1190,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { - stDebug("s-task:%s dispatch transtate msgId:%d to downstream successfully, start to prepare transfer state", id, msgId); + stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", id, msgId); ASSERT(pTask->info.fillHistory == 1); code = streamTransferStatePrepare(pTask); @@ -1048,7 +1202,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i // now ready for next data output atomic_store_8(&pTask->outputq.status, TASK_OUTPUT_STATUS__NORMAL); } else { - handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId); + handleDispatchSuccessRsp(pTask, pRsp->downstreamTaskId, pRsp->downstreamNodeId); } } } @@ -1109,7 +1263,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S stDebug("s-task:%s receive dispatch msg from taskId:0x%x(vgId:%d), msgLen:%" PRId64 ", msgId:%d", id, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->totalLen, pReq->msgId); - SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pReq->upstreamTaskId); ASSERT(pInfo != NULL); if (pMeta->role == NODE_ROLE_FOLLOWER) { @@ -1130,25 +1284,10 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S // This task has received the checkpoint req from the upstream task, from which all the messages should be // blocked. Note that there is no race condition here. if (pReq->type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId); stDebug("s-task:%s close inputQ for upstream:0x%x, msgId:%d", id, pReq->upstreamTaskId, pReq->msgId); } else if (pReq->type == STREAM_INPUT__TRANS_STATE) { - atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); - streamTaskCloseUpstreamInput(pTask, pReq->upstreamTaskId); - - // disable the related stream task here to avoid it to receive the newly arrived data after the transfer-state - STaskId* pRelTaskId = &pTask->streamTaskId; - SStreamTask* pStreamTask = streamMetaAcquireTask(pMeta, pRelTaskId->streamId, pRelTaskId->taskId); - if (pStreamTask != NULL) { - atomic_add_fetch_32(&pStreamTask->upstreamInfo.numOfClosed, 1); - streamTaskCloseUpstreamInput(pStreamTask, pReq->upstreamRelTaskId); - streamMetaReleaseTask(pMeta, pStreamTask); - } - - stDebug("s-task:%s close inputQ for upstream:0x%x since trans-state msgId:%d recv, rel stream-task:0x%" PRIx64 - " close inputQ for upstream:0x%x", - id, pReq->upstreamTaskId, pReq->msgId, pTask->streamTaskId.taskId, pReq->upstreamRelTaskId); + stDebug("s-task:%s recv trans-state msgId:%d from upstream:0x%x", id, pReq->msgId, pReq->upstreamTaskId); } status = streamTaskAppendInputBlocks(pTask, pReq); @@ -1156,11 +1295,6 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } } - // disable the data from upstream tasks -// if (streamTaskGetStatus(pTask)->state == TASK_STATUS__HALT) { -// status = TASK_INPUT_STATUS__BLOCKED; -// } - { // do send response with the input status int32_t code = buildDispatchRsp(pTask, pReq, status, &pRsp->pCont); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 047b169ec9..1828409f89 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -24,6 +24,7 @@ #define FILL_HISTORY_TASK_EXEC_INTERVAL 5000 // 5 sec static int32_t streamTransferStateDoPrepare(SStreamTask* pTask); +static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks); bool streamTaskShouldStop(const SStreamTask* pTask) { SStreamTaskState* pState = streamTaskGetStatus(pTask); @@ -87,8 +88,7 @@ static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* return code; } -static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, - int32_t* totalBlocks) { +int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, int64_t* totalSize, int32_t* totalBlocks) { int32_t code = TSDB_CODE_SUCCESS; void* pExecutor = pTask->exec.pExecutor; @@ -96,7 +96,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i *totalSize = 0; int32_t size = 0; - int32_t numOfBlocks = 0; + int32_t numOfBlocks= 0; SArray* pRes = NULL; while (1) { @@ -184,16 +184,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i return code; } -static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { +static int32_t handleSanhistoryResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { int32_t code = TSDB_CODE_SUCCESS; if (taosArrayGetSize(pRes) > 0) { SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); code = doOutputResultBlockImpl(pTask, pStreamBlocks); - if (code != TSDB_CODE_SUCCESS) { - stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); + if (code != TSDB_CODE_SUCCESS) { // should not have error code + stError("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); } } else { - taosArrayDestroy(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); } return code; } @@ -268,6 +268,17 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { return buildScanhistoryExecRet(TASK_SCANHISTORY_QUIT, 0); } + // output queue is full, idle for 5 sec. + if (streamQueueIsFull(pTask->outputq.queue)) { + stWarn("s-task:%s outputQ is full, idle for 1sec and retry", id); + return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, STREAM_SCAN_HISTORY_TIMESLICE); + } + + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + stWarn("s-task:%s downstream task inputQ blocked, idle for 5sec and retry", id); + return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL); + } + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -284,19 +295,13 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { } // dispatch the generated results - /*int32_t code = */handleResultBlocks(pTask, pRes, size); - - int64_t el = taosGetTimestampMs() - st; - - // downstream task input queue is full, try in 5sec - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED && (pTask->info.fillHistory == 1)) { - return buildScanhistoryExecRet(TASK_SCANHISTORY_REXEC, FILL_HISTORY_TASK_EXEC_INTERVAL); - } + /*int32_t code = */handleSanhistoryResultBlocks(pTask, pRes, size); if (finished) { return buildScanhistoryExecRet(TASK_SCANHISTORY_CONT, 0); } + int64_t el = taosGetTimestampMs() - st; if (el >= STREAM_SCAN_HISTORY_TIMESLICE && (pTask->info.fillHistory == 1)) { stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", id, pTask->info.fillHistory, el / 1000.0); @@ -421,7 +426,7 @@ int32_t streamTransferStatePrepare(SStreamTask* pTask) { streamMetaReleaseTask(pMeta, pStreamTask); return code; } else { - stDebug("s-task:%s halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); + stDebug("s-task:%s sink task halt by related fill-history task:%s", pStreamTask->id.idStr, pTask->id.idStr); } streamMetaReleaseTask(pMeta, pStreamTask); } @@ -536,12 +541,81 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc //static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; } +static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock) { + const char* id = pTask->id.idStr; + int32_t blockSize = 0; + int64_t st = taosGetTimestampMs(); + SCheckpointInfo* pInfo = &pTask->chkInfo; + int64_t ver = pInfo->processedVer; + + stDebug("s-task:%s start to process batch of blocks, num:%d, type:%s", id, 1, "checkpoint-trigger"); + + doSetStreamInputBlock(pTask, pBlock, &ver, id); + + int64_t totalSize = 0; + int32_t totalBlocks = 0; + streamTaskExecImpl(pTask, pBlock, &totalSize, &totalBlocks); + + double el = (taosGetTimestampMs() - st) / 1000.0; + stDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", id, el, + SIZE_IN_MiB(totalSize), totalBlocks); + + pTask->execInfo.outputDataBlocks += totalBlocks; + pTask->execInfo.outputDataSize += totalSize; + if (fabs(el - 0.0) <= DBL_EPSILON) { + pTask->execInfo.procsThroughput = 0; + pTask->execInfo.outputThroughput = 0; + } else { + pTask->execInfo.outputThroughput = (totalSize / el); + pTask->execInfo.procsThroughput = (blockSize / el); + } + + // update the currentVer if processing the submit blocks. + ASSERT(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer); + + if (ver != pInfo->processedVer) { + stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 + " ckpt:%" PRId64, + id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); + pInfo->processedVer = ver; + } +} + +void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointBlock) { + const char* id = pTask->id.idStr; + + // 1. transfer the ownership of executor state + bool dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT); + if (dropRelHTask) { + ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); + + STaskId* pHTaskId = &pTask->hTaskInfo.id; + SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId); + if (pHTask != NULL) { + streamTaskReleaseState(pHTask); + streamTaskReloadState(pTask); + stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, + streamTaskGetStatus(pHTask)->name); + + streamMetaReleaseTask(pTask->pMeta, pHTask); + } else { + stError("s-task:%s related fill-history task:0x%x failed to acquire, transfer state failed", id, + (int32_t)pHTaskId->taskId); + } + } else { + stDebug("s-task:%s no transfer-state needed", id); + } + + // 2. flush data in executor to K/V store, which should be completed before do checkpoint in the K/V. + doStreamTaskExecImpl(pTask, pCheckpointBlock); +} + /** * todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the * appropriate batch of blocks should be handled in 5 to 10 sec. */ static int32_t doStreamExecTask(SStreamTask* pTask) { - const char* id = pTask->id.idStr; + const char* id = pTask->id.idStr; // merge multiple input data if possible in the input queue. stDebug("s-task:%s start to extract data block from inputQ", id); @@ -557,13 +631,13 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (streamQueueIsFull(pTask->outputq.queue)) { - stWarn("s-task:%s outputQ is full, idle for 500ms and retry", id); - streamTaskSetIdleInfo(pTask, 500); + stTrace("s-task:%s outputQ is full, idle for 500ms and retry", id); + streamTaskSetIdleInfo(pTask, 1000); return 0; } if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { - stWarn("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", id); + stTrace("s-task:%s downstream task inputQ blocked, idle for 1sec and retry", id); streamTaskSetIdleInfo(pTask, 1000); return 0; } @@ -586,10 +660,13 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } } + pTask->execInfo.inputDataBlocks += numOfBlocks; + pTask->execInfo.inputDataSize += blockSize; + // dispatch checkpoint msg to all downstream tasks int32_t type = pInput->type; if (type == STREAM_INPUT__CHECKPOINT_TRIGGER) { - streamProcessCheckpointBlock(pTask, (SStreamDataBlock*)pInput); + streamProcessCheckpointTriggerBlock(pTask, (SStreamDataBlock*)pInput); continue; } @@ -601,62 +678,27 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (pTask->info.taskLevel == TASK_LEVEL__SINK) { ASSERT(type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__CHECKPOINT); + int64_t st = taosGetTimestampMs(); + // here only handle the data block sink operation if (type == STREAM_INPUT__DATA_BLOCK) { pTask->execInfo.sink.dataSize += blockSize; stDebug("s-task:%s sink task start to sink %d blocks, size:%.2fKiB", id, numOfBlocks, SIZE_IN_KiB(blockSize)); doOutputResultBlockImpl(pTask, (SStreamDataBlock*)pInput); + + double el = (taosGetTimestampMs() - st) / 1000.0; + if (fabs(el - 0.0) <= DBL_EPSILON) { + pTask->execInfo.procsThroughput = 0; + } else { + pTask->execInfo.procsThroughput = (blockSize / el); + } + continue; } } - if (type == STREAM_INPUT__CHECKPOINT) { - // transfer the state from fill-history to related stream task before generating the checkpoint. - bool dropRelHTask = (streamTaskGetPrevStatus(pTask) == TASK_STATUS__HALT); - if (dropRelHTask) { - ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); - - STaskId* pHTaskId = &pTask->hTaskInfo.id; - SStreamTask* pHTask = streamMetaAcquireTask(pTask->pMeta, pHTaskId->streamId, pHTaskId->taskId); - if (pHTask != NULL) { - // 2. transfer the ownership of executor state - streamTaskReleaseState(pHTask); - streamTaskReloadState(pTask); - stDebug("s-task:%s transfer state from fill-history task:%s, status:%s completed", id, pHTask->id.idStr, - streamTaskGetStatus(pHTask)->name); - - streamMetaReleaseTask(pTask->pMeta, pHTask); - } else { - stError("s-task:%s related fill-history task:0x%x failed to acquire, transfer state failed", id, - (int32_t)pHTaskId->taskId); - } - } - } - - int64_t st = taosGetTimestampMs(); - stDebug("s-task:%s start to process batch of blocks, num:%d, type:%s", id, numOfBlocks, streamQueueItemGetTypeStr(type)); - - int64_t ver = pTask->chkInfo.processedVer; - doSetStreamInputBlock(pTask, pInput, &ver, id); - - int64_t resSize = 0; - int32_t totalBlocks = 0; - streamTaskExecImpl(pTask, pInput, &resSize, &totalBlocks); - - double el = (taosGetTimestampMs() - st) / 1000.0; - stDebug("s-task:%s batch of input blocks exec end, elapsed time:%.2fs, result size:%.2fMiB, numOfBlocks:%d", id, el, - SIZE_IN_MiB(resSize), totalBlocks); - - SCheckpointInfo* pInfo = &pTask->chkInfo; - - // update the currentVer if processing the submit blocks. - ASSERT(pInfo->checkpointVer <= pInfo->nextProcessVer && ver >= pInfo->checkpointVer); - - if (ver != pInfo->processedVer) { - stDebug("s-task:%s update processedVer(unsaved) from %" PRId64 " to %" PRId64 " nextProcessVer:%" PRId64 - " ckpt:%" PRId64, - id, pInfo->processedVer, ver, pInfo->nextProcessVer, pInfo->checkpointVer); - pInfo->processedVer = ver; + if (type != STREAM_INPUT__CHECKPOINT) { + doStreamTaskExecImpl(pTask, pInput); } streamFreeQitem(pInput); @@ -664,7 +706,6 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { - // todo add lock SStreamTaskState* pState = streamTaskGetStatus(pTask); if (pState->state == TASK_STATUS__CK) { @@ -689,8 +730,6 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { return 0; } } - - return 0; } // the task may be set dropping/stopping, while it is still in the task queue, therefore, the sched-status can not diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0aace1cb5b..f6449829a3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -151,10 +151,10 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { int8_t ret = STREAM_STATA_COMPATIBLE; TBC* pCur = NULL; - if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { - // no task info, no stream + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream return ret; } + void* pKey = NULL; int32_t kLen = 0; void* pVal = NULL; @@ -165,20 +165,24 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { if (pVal == NULL || vLen == 0) { break; } + SDecoder decoder; SCheckpointInfo info; tDecoderInit(&decoder, (uint8_t*)pVal, vLen); if (tDecodeStreamTaskChkInfo(&decoder, &info) < 0) { continue; } + if (info.msgVer <= SSTREAM_TASK_INCOMPATIBLE_VER) { ret = STREAM_STATA_NO_COMPATIBLE; } else if (info.msgVer >= SSTREAM_TASK_NEED_CONVERT_VER) { ret = STREAM_STATA_NEED_CONVERT; } + tDecoderClear(&decoder); break; } + tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); @@ -193,6 +197,7 @@ int32_t streamMetaCvtDbFormat(SStreamMeta* pMeta) { if (exist == false) { return code; } + SBackendWrapper* pBackend = streamBackendInit(pMeta->path, chkpId, pMeta->vgId); void* pIter = taosHashIterate(pBackend->cfInst, NULL); @@ -219,6 +224,7 @@ _EXIT: return code; } + int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { int8_t compatible = streamMetaCheckBackendCompatible(pMeta); if (compatible == STREAM_STATA_COMPATIBLE) { @@ -239,14 +245,12 @@ int32_t streamMetaMayCvtDbFormat(SStreamMeta* pMeta) { return 0; } -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { - SStreamTask* pTask = arg; - +int32_t streamTaskSetDb(SStreamMeta* pMeta, SStreamTask* pTask, const char* key) { int64_t chkpId = pTask->chkInfo.checkpointId; taosThreadMutexLock(&pMeta->backendMutex); void** ppBackend = taosHashGet(pMeta->pTaskDbUnique, key, strlen(key)); - if (ppBackend != NULL && *ppBackend != NULL) { + if ((ppBackend != NULL) && (*ppBackend != NULL)) { taskDbAddRef(*ppBackend); STaskDbWrapper* pBackend = *ppBackend; @@ -254,24 +258,22 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { pTask->pBackend = pBackend; taosThreadMutexUnlock(&pMeta->backendMutex); - stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } - STaskDbWrapper* pBackend = taskDbOpen(pMeta->path, key, chkpId); + STaskDbWrapper* pBackend = NULL; while (1) { - if (pBackend == NULL) { - taosThreadMutexUnlock(&pMeta->backendMutex); - taosMsleep(1000); - stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); - } else { - taosThreadMutexUnlock(&pMeta->backendMutex); + pBackend = taskDbOpen(pMeta->path, key, chkpId); + if (pBackend != NULL) { break; } + taosThreadMutexUnlock(&pMeta->backendMutex); + taosMsleep(1000); + + stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); taosThreadMutexLock(&pMeta->backendMutex); - pBackend = taskDbOpen(pMeta->path, key, chkpId); } int64_t tref = taosAddRef(taskDbWrapperId, pBackend); @@ -456,10 +458,10 @@ void streamMetaClear(SStreamMeta* pMeta) { SStreamTask* p = *(SStreamTask**)pIter; // release the ref by timer - if (p->info.triggerParam != 0 && p->info.fillHistory == 0) { // one more ref in timer + if (p->info.delaySchedParam != 0 && p->info.fillHistory == 0) { // one more ref in timer stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", p->id.idStr, p->refCnt); taosTmrStop(p->schedInfo.pDelayTimer); - p->info.triggerParam = 0; + p->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, p); } @@ -541,6 +543,7 @@ void streamMetaCloseImpl(void* arg) { // todo let's check the status for each task int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { + int32_t vgId = pTask->pMeta->vgId; void* buf = NULL; int32_t len; int32_t code; @@ -556,19 +559,23 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ pTask->ver = SSTREAM_TASK_VER; } + SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); int64_t id[2] = {pTask->id.streamId, pTask->id.taskId}; - if (tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn) < 0) { - stError("s-task:%s save to disk failed, code:%s", pTask->id.idStr, tstrerror(terrno)); - return -1; + + code = tdbTbUpsert(pMeta->pTaskDb, id, STREAM_TASK_KEY_LEN, buf, len, pMeta->txn); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s vgId:%d task meta save to disk failed, code:%s", pTask->id.idStr, vgId, tstrerror(terrno)); + } else { + stDebug("s-task:%s vgId:%d task meta save to disk", pTask->id.idStr, vgId); } taosMemoryFree(buf); - return 0; + return code; } int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pTaskId) { @@ -745,10 +752,10 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ASSERT(pTask->status.timerActive == 0); - if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) { + if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); taosTmrStop(pTask->schedInfo.pDelayTimer); - pTask->info.triggerParam = 0; + pTask->info.delaySchedParam = 0; streamMetaReleaseTask(pMeta, pTask); } @@ -791,11 +798,11 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) { } int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { - int64_t chkpId = 0; + int64_t checkpointId = 0; TBC* pCur = NULL; if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { - return chkpId; + return checkpointId; } void* pKey = NULL; @@ -816,16 +823,16 @@ int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { } tDecoderClear(&decoder); - chkpId = TMAX(chkpId, info.checkpointId); + checkpointId = TMAX(checkpointId, info.checkpointId); } - stDebug("get max chkp id: %" PRId64 "", chkpId); + stDebug("vgId:%d get max checkpointId:%" PRId64, pMeta->vgId, checkpointId); tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); - return chkpId; + return checkpointId; } // not allowed to return error code @@ -854,6 +861,10 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { tdbTbcMoveToFirst(pCur); while (tdbTbcNext(pCur, &pKey, &kLen, &pVal, &vLen) == 0) { + if (pVal == NULL || vLen == 0) { + break; + } + SStreamTask* pTask = taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1017,8 +1028,13 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .checkpointInfo.latestId = (*pTask)->chkInfo.checkpointId, .checkpointInfo.latestVer = (*pTask)->chkInfo.checkpointVer, .checkpointInfo.latestTime = (*pTask)->chkInfo.checkpointTime, + .checkpointInfo.latestSize = 0, + .checkpointInfo.remoteBackup = 0, .hTaskId = (*pTask)->hTaskInfo.id.taskId, - + .procsTotal = SIZE_IN_MiB((*pTask)->execInfo.inputDataSize), + .outputTotal = SIZE_IN_MiB((*pTask)->execInfo.outputDataSize), + .procsThroughput = SIZE_IN_KiB((*pTask)->execInfo.procsThroughput), + .outputThroughput = SIZE_IN_KiB((*pTask)->execInfo.outputThroughput), .startCheckpointId = (*pTask)->execInfo.startCheckpointId, .startCheckpointVer = (*pTask)->execInfo.startCheckpointVer, }; @@ -1029,13 +1045,13 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { entry.sinkDataSize = SIZE_IN_MiB((*pTask)->execInfo.sink.dataSize); } - if ((*pTask)->chkInfo.checkpointingId != 0) { - entry.checkpointInfo.failed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId) ? 1 : 0; - entry.checkpointInfo.activeId = (*pTask)->chkInfo.checkpointingId; - entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.transId; + if ((*pTask)->chkInfo.pActiveInfo->activeId != 0) { + entry.checkpointInfo.failed = ((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0; + entry.checkpointInfo.activeId = (*pTask)->chkInfo.pActiveInfo->activeId; + entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.pActiveInfo->transId; if (entry.checkpointInfo.failed) { - stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); + stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.pActiveInfo->transId); } } @@ -1105,6 +1121,7 @@ void metaHbToMnode(void* param, void* tmrId) { SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); if (pMeta == NULL) { + stError("invalid rid:%" PRId64 " failed to acquired stream-meta", rid); return; } @@ -1325,8 +1342,8 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) streamMetaWUnLock(pMeta); if (isLeader) { - stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, - prevStage, stage, isLeader); + stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb, rid:%" PRId64, + pMeta->vgId, prevStage, stage, isLeader, pMeta->rid); streamMetaStartHb(pMeta); } else { stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, @@ -1355,7 +1372,7 @@ static int32_t prepareBeforeStartTasks(SStreamMeta* pMeta, SArray** pList, int64 return TSDB_CODE_SUCCESS; } -int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expandFn) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = pMeta->vgId; int64_t now = taosGetTimestampMs(); @@ -1375,13 +1392,36 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { return TSDB_CODE_SUCCESS; } + // broadcast the check downstream tasks msg numOfTasks = taosArrayGetSize(pTaskList); - // broadcast the check downstream tasks msg + // prepare the fill-history task before starting all stream tasks, to avoid fill-history tasks are started without + // initialization , when the operation of check downstream tasks status is executed far quickly. + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); + streamMetaAddFailedTask(pMeta, pTaskId->streamId, pTaskId->taskId); + continue; + } + + if (pTask->pBackend == NULL) { // TODO: add test cases for this + code = expandFn(pTask); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:0x%x vgId:%d failed to expand stream backend", pTaskId->taskId, vgId); + streamMetaAddFailedTaskSelf(pTask, pTask->execInfo.readyTs); + } + } else { + stDebug("s-task:0x%x vgId:%d fill-history task backend has initialized already", pTaskId->taskId, vgId); + } + + streamMetaReleaseTask(pMeta, pTask); + } + for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - // todo: use hashTable instead SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); @@ -1389,14 +1429,16 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { continue; } - // fill-history task can only be launched by related stream tasks. STaskExecStatisInfo* pInfo = &pTask->execInfo; + + // fill-history task can only be launched by related stream tasks. if (pTask->info.fillHistory == 1) { stDebug("s-task:%s fill-history task wait related stream task start", pTask->id.idStr); streamMetaReleaseTask(pMeta, pTask); continue; } + // ready now, start the related fill-history task if (pTask->status.downstreamReady == 1) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", @@ -1419,7 +1461,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamMetaReleaseTask(pMeta, pTask); } - stInfo("vgId:%d start tasks completed", pMeta->vgId); + stInfo("vgId:%d start all task(s) completed", pMeta->vgId); taosArrayDestroy(pTaskList); return code; } @@ -1478,13 +1520,13 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { return true; } -int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { +int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, __stream_task_expand_fn expandFn) { int32_t vgId = pMeta->vgId; - stInfo("vgId:%d start to task:0x%x by checking downstream status", vgId, taskId); + stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, streamId, taskId); if (pTask == NULL) { - stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, taskId); + stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); streamMetaAddFailedTask(pMeta, streamId, taskId); return TSDB_CODE_STREAM_TASK_IVLD_STATUS; } @@ -1497,6 +1539,32 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas } ASSERT(pTask->status.downstreamReady == 0); + if (pTask->pBackend == NULL) { + int32_t code = expandFn(pTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); + streamMetaReleaseTask(pMeta, pTask); + return code; + } + + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); + if (pHTask != NULL) { + if (pHTask->pBackend == NULL) { + code = expandFn(pHTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaAddFailedTaskSelf(pHTask, pInfo->readyTs); + + streamMetaReleaseTask(pMeta, pHTask); + streamMetaReleaseTask(pMeta, pTask); + return code; + } + } + + streamMetaReleaseTask(pMeta, pHTask); + } + } + } int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); if (ret != TSDB_CODE_SUCCESS) { diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 5596eb3dee..9c5c230a3d 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -117,6 +117,13 @@ int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue) { return numOfItems1 + numOfItems2; } +int32_t streamQueueGetNumOfUnAccessedItems(const SStreamQueue* pQueue) { + int32_t numOfItems1 = taosQueueItemSize(pQueue->pQueue); + int32_t numOfItems2 = taosQallUnAccessedItemSize(pQueue->qall); + + return numOfItems1 + numOfItems2; +} + int32_t streamQueueGetItemSize(const SStreamQueue* pQueue) { return taosQueueMemorySize(pQueue->pQueue) + taosQallUnAccessedMemSize(pQueue->qall); } @@ -322,9 +329,10 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) ASSERT(0); } - if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->info.triggerParam != 0) { + if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && type != STREAM_INPUT__CHECKPOINT_TRIGGER && + (pTask->info.delaySchedParam != 0)) { atomic_val_compare_exchange_8(&pTask->schedInfo.status, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); - stDebug("s-task:%s new data arrived, active the trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status); + stDebug("s-task:%s new data arrived, active the sched-trigger, triggerStatus:%d", pTask->id.idStr, pTask->schedInfo.status); } return 0; @@ -381,7 +389,7 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc } } - return TSDB_CODE_SUCCESS; + return code; } int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 52e7431e70..9c817d565b 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -20,13 +20,13 @@ static void streamTaskResumeHelper(void* param, void* tmrId); static void streamTaskSchedHelper(void* param, void* tmrId); int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { - if (pTask->info.triggerParam != 0 && pTask->info.fillHistory == 0) { + if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1); ASSERT(ref == 2 && pTask->schedInfo.pDelayTimer == NULL); - stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.triggerParam); + stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam); - pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.triggerParam, pTask, streamTimer); + pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } @@ -52,7 +52,11 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3 return -1; } - stDebug("vgId:%d create msg to start stream task:0x%x, exec type:%d", vgId, taskId, execType); + if (streamId != 0) { + stDebug("vgId:%d create msg to start stream task:0x%x, exec type:%d", vgId, taskId, execType); + } else { + stDebug("vgId:%d create msg to exec, type:%d", vgId, execType); + } pRunReq->head.vgId = vgId; pRunReq->streamId = streamId; @@ -115,7 +119,7 @@ void streamTaskResumeHelper(void* param, void* tmrId) { void streamTaskSchedHelper(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; const char* id = pTask->id.idStr; - int32_t nextTrigger = (int32_t)pTask->info.triggerParam; + int32_t nextTrigger = (int32_t)pTask->info.delaySchedParam; int8_t status = atomic_load_8(&pTask->schedInfo.status); stTrace("s-task:%s in scheduler, trigger status:%d, next:%dms", id, status, nextTrigger); diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 687b4bcf12..4c61e6da1d 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -734,6 +734,21 @@ _end: return code; } +int32_t getCountWinStateFromDisc(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen) { + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentNext_rocksdb(pState, pKey); + int32_t code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); + streamStateFreeCur(pCur); + if (code == TSDB_CODE_SUCCESS) { + return code; + } else { + pCur = streamStateSessionSeekKeyPrev_rocksdb(pState, pKey); + } + + code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, pVal, pVLen); + streamStateFreeCur(pCur); + return code; +} + int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, COUNT_TYPE winCount, void** pVal, int32_t* pVLen) { SSessionKey* pWinKey = pKey; const TSKEY gap = 0; @@ -755,14 +770,13 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C if (size == 0) { void* pFileStore = getStateFileStore(pFileState); void* pRockVal = NULL; - SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId); - code = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &pRockVal, pVLen); - streamStateFreeCur(pCur); + code = getCountWinStateFromDisc(pFileStore, pWinKey, &pRockVal, pVLen); if (code == TSDB_CODE_SUCCESS || isFlushedState(pFileState, endTs, 0)) { - qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code); + qDebug("===stream===0 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code); if (code == TSDB_CODE_SUCCESS) { int32_t valSize = *pVLen; - COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)( (char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE)) ); + COUNT_TYPE* pWinStateCout = (COUNT_TYPE*)((char*)(pRockVal) + (valSize - sizeof(COUNT_TYPE))); if (inSessionWindow(pWinKey, startTs, gap) || (*pWinStateCout) < winCount) { (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); goto _end; @@ -798,20 +812,24 @@ int32_t getCountWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, C } if (index == -1) { - if (!isDeteled(pFileState, endTs)) { - void* p = NULL; - void* pFileStore = getStateFileStore(pFileState); - SStreamStateCur* pCur = streamStateSessionSeekToLast_rocksdb(pFileStore, pKey->groupId); - int32_t code_file = streamStateSessionGetKVByCur_rocksdb(pCur, pWinKey, &p, pVLen); + if (!isDeteled(pFileState, endTs) && isFlushedState(pFileState, endTs, 0)) { + SSessionKey tmpKey = *pWinKey; + void* pRockVal = NULL; + void* pFileStore = getStateFileStore(pFileState); + int32_t code_file = getCountWinStateFromDisc(pFileStore, &tmpKey, &pRockVal, pVLen); if (code_file == TSDB_CODE_SUCCESS) { - (*pVal) = createSessionWinBuff(pFileState, pWinKey, p, pVLen); - code = code_file; - qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, pWinKey->win.ekey, code_file); - streamStateFreeCur(pCur); - goto _end; + SRowBuffPos* pFirstPos = taosArrayGetP(pWinStates, 0); + SSessionKey* pFirstWinKey = (SSessionKey*)pFirstPos->pKey; + if (tmpKey.win.ekey < pFirstWinKey->win.skey) { + *pWinKey = tmpKey; + (*pVal) = createSessionWinBuff(pFileState, pWinKey, pRockVal, pVLen); + code = code_file; + qDebug("===stream===1 get state win:%" PRId64 ",%" PRId64 " from disc, res %d", pWinKey->win.skey, + pWinKey->win.ekey, code_file); + goto _end; + } } - taosMemoryFree(p); - streamStateFreeCur(pCur); + taosMemoryFree(pRockVal); } } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 1800324cc8..adefe97f1f 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -74,7 +74,9 @@ struct SStreamSnapHandle { int32_t currFileIdx; char* metaPath; + void* pMeta; SArray* pDbSnapSet; + SArray* pSnapInfoSet; int32_t currIdx; int8_t delFlag; // 0 : not del, 1: del }; @@ -140,7 +142,9 @@ TdFilePtr streamOpenFile(char* path, char* name, int32_t opt) { return taosOpenFile(fullname, opt); } -int32_t streamTaskDbGetSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); } +int32_t streamCreateTaskDbSnapInfo(void* arg, char* path, SArray* pSnap) { return taskDbBuildSnap(arg, pSnap); } + +int32_t streamDestroyTaskDbSnapInfo(void* arg, SArray* snap) { return taskDbDestroySnap(arg, snap); } void snapFileDebugInfo(SBackendSnapFile2* pSnapFile) { if (qDebugFlag & DEBUG_DEBUG) { @@ -291,30 +295,27 @@ void snapFileDestroy(SBackendSnapFile2* pSnap) { } int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, void* pMeta) { // impl later - SArray* pSnapSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); - int32_t code = streamTaskDbGetSnapInfo(pMeta, path, pSnapSet); + SArray* pSnapInfoSet = taosArrayInit(4, sizeof(SStreamTaskSnap)); + int32_t code = streamCreateTaskDbSnapInfo(pMeta, path, pSnapInfoSet); if (code != 0) { + taosArrayDestroy(pSnapInfoSet); return -1; } SArray* pDbSnapSet = taosArrayInit(8, sizeof(SBackendSnapFile2)); - for (int32_t i = 0; i < taosArrayGetSize(pSnapSet); i++) { - SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); + for (int32_t i = 0; i < taosArrayGetSize(pSnapInfoSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(pSnapInfoSet, i); SBackendSnapFile2 snapFile = {0}; code = streamBackendSnapInitFile(path, pSnap, &snapFile); ASSERT(code == 0); taosArrayPush(pDbSnapSet, &snapFile); } - for (int32_t i = 0; i < taosArrayGetSize(pSnapSet); i++) { - SStreamTaskSnap* pSnap = taosArrayGet(pSnapSet, i); - taosMemoryFree(pSnap->dbPrefixPath); - } - taosArrayDestroy(pSnapSet); - pHandle->pDbSnapSet = pDbSnapSet; + pHandle->pSnapInfoSet = pSnapInfoSet; pHandle->currIdx = 0; + pHandle->pMeta = pMeta; return 0; _err: @@ -333,6 +334,14 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { } taosArrayDestroy(handle->pDbSnapSet); } + streamDestroyTaskDbSnapInfo(handle->pMeta, handle->pSnapInfoSet); + if (handle->pSnapInfoSet) { + for (int32_t i = 0; i < taosArrayGetSize(handle->pSnapInfoSet); i++) { + SStreamTaskSnap* pSnap = taosArrayGet(handle->pSnapInfoSet, i); + taosMemoryFree(pSnap->dbPrefixPath); + } + taosArrayDestroy(handle->pSnapInfoSet); + } taosMemoryFree(handle->metaPath); return; } diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 6882f6617d..7a864a60d2 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -155,7 +155,7 @@ int32_t streamTaskOnNormalTaskReady(SStreamTask* pTask) { return TSDB_CODE_SUCCESS; } -int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { +int32_t streamTaskOnScanHistoryTaskReady(SStreamTask* pTask) { // set the state to be ready streamTaskSetReady(pTask); streamTaskSetRangeStreamCalc(pTask); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 52002b7ea8..47324bd8c9 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -98,7 +98,7 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { return winKeyCmprImpl(&pWin1->key, &pWin2->key); } -SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { +SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); if (pState == NULL) { @@ -127,7 +127,7 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT); pState->parNameMap = tSimpleHashInit(1024, hashFn); - stInfo("succ to open state %p on backend %p 0x%" PRIx64 "-%d", pState, pMeta->streamBackend, pState->streamId, + stInfo("open state %p on backend %p 0x%" PRIx64 "-%d succ", pState, pMeta->streamBackend, pState->streamId, pState->taskId); return pState; diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 1e622f615d..dc9d2166e6 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -24,6 +24,7 @@ static void streamTaskDestroyUpstreamInfo(SUpstreamInfo* pUpstreamInfo); static void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated); static void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdate); +static void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo); static int32_t addToTaskset(SArray* pArray, SStreamTask* pTask) { int32_t childId = taosArrayGetSize(pArray); @@ -70,12 +71,12 @@ static void freeItem(void* p) { } static void freeUpstreamItem(void* p) { - SStreamChildEpInfo** pInfo = p; + SStreamUpstreamEpInfo** pInfo = p; taosMemoryFree(*pInfo); } -static SStreamChildEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) { - SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); +static SStreamUpstreamEpInfo* createStreamTaskEpInfo(const SStreamTask* pTask) { + SStreamUpstreamEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamUpstreamEpInfo)); if (pEpInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -106,7 +107,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, pTask->info.taskLevel = taskLevel; pTask->info.fillHistory = fillHistory; - pTask->info.triggerParam = triggerParam; + pTask->info.delaySchedParam = triggerParam; pTask->subtableWithoutMd5 = subtableWithoutMd5; pTask->status.pSM = streamCreateStateMachine(pTask); @@ -187,8 +188,9 @@ int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId) { } void tFreeStreamTask(SStreamTask* pTask) { - char* p = NULL; - int32_t taskId = pTask->id.taskId; + char* p = NULL; + int32_t taskId = pTask->id.taskId; + STaskExecStatisInfo* pStatis = &pTask->execInfo; ETaskStatus status1 = TASK_STATUS__UNINIT; @@ -200,7 +202,7 @@ void tFreeStreamTask(SStreamTask* pTask) { } taosThreadMutexUnlock(&pTask->lock); - stDebug("start to free s-task:0x%x, %p, state:%s", taskId, pTask, p); + stDebug("start to free s-task:0x%x %p, state:%s", taskId, pTask, p); SCheckpointInfo* pCkInfo = &pTask->chkInfo; stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64 @@ -226,9 +228,9 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->hTaskInfo.pTimer = NULL; } - if (pTask->msgInfo.pTimer != NULL) { - /*bool ret = */taosTmrStop(pTask->msgInfo.pTimer); - pTask->msgInfo.pTimer = NULL; + if (pTask->msgInfo.pRetryTmr != NULL) { + /*bool ret = */taosTmrStop(pTask->msgInfo.pRetryTmr); + pTask->msgInfo.pRetryTmr = NULL; } if (pTask->inputq.queue) { @@ -253,7 +255,6 @@ void tFreeStreamTask(SStreamTask* pTask) { } streamClearChkptReadyMsg(pTask); - pTask->pReadyMsgList = taosArrayDestroy(pTask->pReadyMsgList); if (pTask->msgInfo.pData != NULL) { clearBufferedDispatchMsg(pTask); @@ -275,10 +276,6 @@ void tFreeStreamTask(SStreamTask* pTask) { taskDbRemoveRef(pTask->pBackend); } - if (pTask->id.idStr != NULL) { - taosMemoryFree((void*)pTask->id.idStr); - } - if (pTask->pNameMap) { tSimpleHashCleanup(pTask->pNameMap); } @@ -292,6 +289,22 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); + if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) { + char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128); + sprintf(path, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, pTask->id.idStr); + taosRemoveDir(path); + + stInfo("s-task:0x%x vgId:%d remove all backend files:%s", taskId, pTask->pMeta->vgId, path); + taosMemoryFree(path); + } + + if (pTask->id.idStr != NULL) { + taosMemoryFree((void*)pTask->id.idStr); + } + + streamTaskDestroyActiveChkptInfo(pTask->chkInfo.pActiveInfo); + pTask->chkInfo.pActiveInfo = NULL; + taosMemoryFree(pTask); stDebug("s-task:0x%x free task completed", taskId); } @@ -404,6 +417,10 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i return TSDB_CODE_OUT_OF_MEMORY; } + if (pTask->chkInfo.pActiveInfo == NULL) { + pTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); + } + return TSDB_CODE_SUCCESS; } @@ -423,8 +440,12 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) { } } +int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) { + return taosArrayGetSize(pTask->upstreamInfo.pList); +} + int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask) { - SStreamChildEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask); + SStreamUpstreamEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask); if (pEpInfo == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -443,7 +464,7 @@ void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpS int32_t numOfUpstream = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < numOfUpstream; ++i) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); if (pInfo->nodeId == nodeId) { bool equal = isEpsetEqual(&pInfo->epSet, pEpSet); if (!equal) { @@ -579,7 +600,7 @@ void streamTaskResetUpstreamStageInfo(SStreamTask* pTask) { int32_t size = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < size; ++i) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); pInfo->stage = -1; } @@ -593,7 +614,7 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) { } for (int32_t i = 0; i < num; ++i) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); pInfo->dataAllowed = true; } @@ -602,9 +623,20 @@ void streamTaskOpenAllUpstreamInput(SStreamTask* pTask) { } void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId) { - SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); - if (pInfo != NULL) { + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); + if ((pInfo != NULL) && pInfo->dataAllowed) { pInfo->dataAllowed = false; + int32_t t = atomic_add_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); + ASSERT(t <= streamTaskGetNumOfUpstream(pTask)); + } +} + +void streamTaskOpenUpstreamInput(SStreamTask* pTask, int32_t taskId) { + SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, taskId); + if ((pInfo != NULL) && (!pInfo->dataAllowed)) { + int32_t t = atomic_sub_fetch_32(&pTask->upstreamInfo.numOfClosed, 1); + ASSERT(t >= 0); + pInfo->dataAllowed = true; } } @@ -686,16 +718,47 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI pReq->head.vgId = vgId; pReq->taskId = pTaskId->taskId; pReq->streamId = pTaskId->streamId; - pReq->resetRelHalt = resetRelHalt; + pReq->resetRelHalt = resetRelHalt; // todo: remove this attribute SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_DROP, .pCont = pReq, .contLen = sizeof(SVDropStreamTaskReq)}; int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg); if (code != TSDB_CODE_SUCCESS) { stError("vgId:%d failed to send drop task:0x%x msg, code:%s", vgId, pTaskId->taskId, tstrerror(code)); - return code; + } else { + stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId); } - stDebug("vgId:%d build and send drop task:0x%x msg", vgId, pTaskId->taskId); + return code; +} + +int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId, + SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { + SVUpdateCheckpointInfoReq* pReq = rpcMallocCont(sizeof(SVUpdateCheckpointInfoReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pReq->head.vgId = vgId; + pReq->taskId = pTaskId->taskId; + pReq->streamId = pTaskId->streamId; + pReq->dropRelHTask = dropRelHTask; + pReq->hStreamId = pHTaskId->streamId; + pReq->hTaskId = pHTaskId->taskId; + pReq->transId = pCheckpointInfo->pActiveInfo->transId; + + pReq->checkpointId = pCheckpointInfo->pActiveInfo->activeId; + pReq->checkpointVer = pCheckpointInfo->processedVer; + pReq->checkpointTs = pCheckpointInfo->startTs; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_UPDATE_CHKPT, .pCont = pReq, .contLen = sizeof(SVUpdateCheckpointInfoReq)}; + int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg); + + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d task:0x%x failed to send update checkpoint info msg, code:%s", vgId, pTaskId->taskId, tstrerror(code)); + } else { + stDebug("vgId:%d task:0x%x build and send update checkpoint info msg msg", vgId, pTaskId->taskId); + } return code; } @@ -730,6 +793,10 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->stage = pSrc->stage; pDst->inputQUsed = pSrc->inputQUsed; pDst->inputRate = pSrc->inputRate; + pDst->procsTotal = pSrc->procsTotal; + pDst->procsThroughput = pSrc->procsThroughput; + pDst->outputTotal = pSrc->outputTotal; + pDst->outputThroughput = pSrc->outputThroughput; pDst->processedVer = pSrc->processedVer; pDst->verRange = pSrc->verRange; pDst->sinkQuota = pSrc->sinkQuota; @@ -815,10 +882,10 @@ int32_t streamTaskSendCheckpointReq(SStreamTask* pTask) { return 0; } -SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { +SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId) { int32_t num = taosArrayGetSize(pTask->upstreamInfo.pList); for (int32_t i = 0; i < num; ++i) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); if (pInfo->taskId == taskId) { return pInfo; } @@ -828,6 +895,24 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t return NULL; } +SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { + if (pTask->info.taskLevel == TASK_OUTPUT__FIXED_DISPATCH) { + if (pTask->outputInfo.fixedDispatcher.taskId == taskId) { + return &pTask->outputInfo.fixedDispatcher.epSet; + } + } else if (pTask->info.taskLevel == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* pList = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; + for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + SVgroupInfo* pVgInfo = taosArrayGet(pList, i); + if (pVgInfo->taskId == taskId) { + return &pVgInfo->epSet; + } + } + } + + return NULL; +} + char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { char buf[128] = {0}; sprintf(buf, "0x%" PRIx64 "-0x%x", streamId, taskId); @@ -865,4 +950,76 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) { return code; } return streamTrySchedExec(pTask); +} + +void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { + pTask->status.removeBackendFiles = true; +} + +int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { + if (pTransId != NULL) { + *pTransId = pTask->chkInfo.pActiveInfo->transId; + } + + if (pCheckpointId != NULL) { + *pCheckpointId = pTask->chkInfo.pActiveInfo->activeId; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskSetActiveCheckpointInfo(SStreamTask* pTask, int64_t activeCheckpointId) { + pTask->chkInfo.pActiveInfo->activeId = activeCheckpointId; + return TSDB_CODE_SUCCESS; +} + +int32_t streamTaskSetFailedChkptInfo(SStreamTask* pTask, int32_t transId, int64_t checkpointId) { + pTask->chkInfo.pActiveInfo->transId = transId; + pTask->chkInfo.pActiveInfo->activeId = checkpointId; + pTask->chkInfo.pActiveInfo->failedId = checkpointId; + return TSDB_CODE_SUCCESS; +} + +SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo() { + SActiveCheckpointInfo* pInfo = taosMemoryCalloc(1, sizeof(SActiveCheckpointInfo)); + taosThreadMutexInit(&pInfo->lock, NULL); + + pInfo->pDispatchTriggerList = taosArrayInit(4, sizeof(STaskTriggerSendInfo)); + pInfo->pReadyMsgList = taosArrayInit(4, sizeof(STaskCheckpointReadyInfo)); + pInfo->pCheckpointReadyRecvList = taosArrayInit(4, sizeof(STaskDownstreamReadyInfo)); + return pInfo; +} + +void streamTaskDestroyActiveChkptInfo(SActiveCheckpointInfo* pInfo) { + if (pInfo == NULL) { + return; + } + + taosThreadMutexDestroy(&pInfo->lock); + pInfo->pDispatchTriggerList = taosArrayDestroy(pInfo->pDispatchTriggerList); + pInfo->pReadyMsgList = taosArrayDestroy(pInfo->pReadyMsgList); + pInfo->pCheckpointReadyRecvList = taosArrayDestroy(pInfo->pCheckpointReadyRecvList); + + if (pInfo->pChkptTriggerTmr != NULL) { + taosTmrStop(pInfo->pChkptTriggerTmr); + pInfo->pChkptTriggerTmr = NULL; + } + + if (pInfo->pSendReadyMsgTmr != NULL) { + taosTmrStop(pInfo->pSendReadyMsgTmr); + pInfo->pSendReadyMsgTmr = NULL; + } + + taosMemoryFree(pInfo); +} + +void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) { + pInfo->activeId = 0; // clear the checkpoint id + pInfo->failedId = 0; + pInfo->transId = 0; + pInfo->allUpstreamTriggerRecv = 0; + pInfo->dispatchTrigger = false; + + taosArrayClear(pInfo->pDispatchTriggerList); + taosArrayClear(pInfo->pCheckpointReadyRecvList); } \ No newline at end of file diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index cced6a6b84..82ea2f88ef 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -584,7 +584,7 @@ void doInitStateTransferTable(void) { // initialization event handle STaskStateTrans trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__READY, TASK_EVENT_INIT, streamTaskInitStatus, streamTaskOnNormalTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); - trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanhistoryTaskReady, NULL); + trans = createStateTransform(TASK_STATUS__UNINIT, TASK_STATUS__SCAN_HISTORY, TASK_EVENT_INIT_SCANHIST, streamTaskInitStatus, streamTaskOnScanHistoryTaskReady, NULL); taosArrayPush(streamTaskSMTrans, &trans); // scan-history related event diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c index 5e52b927c6..705406f044 100644 --- a/source/libs/stream/src/streammsg.c +++ b/source/libs/stream/src/streammsg.c @@ -17,7 +17,7 @@ #include "streammsg.h" #include "tstream.h" -int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) { +int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; @@ -26,7 +26,7 @@ int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) return 0; } -int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { +int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamUpstreamEpInfo* pInfo) { if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; @@ -335,6 +335,10 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI32(pEncoder, ps->nodeId) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputQUsed) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputRate) < 0) return -1; + if (tEncodeDouble(pEncoder, ps->procsTotal) < 0) return -1; + if (tEncodeDouble(pEncoder, ps->procsThroughput) < 0) return -1; + if (tEncodeDouble(pEncoder, ps->outputTotal) < 0) return -1; + if (tEncodeDouble(pEncoder, ps->outputThroughput) < 0) return -1; if (tEncodeDouble(pEncoder, ps->sinkQuota) < 0) return -1; if (tEncodeDouble(pEncoder, ps->sinkDataSize) < 0) return -1; if (tEncodeI64(pEncoder, ps->processedVer) < 0) return -1; @@ -346,6 +350,8 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI64(pEncoder, ps->checkpointInfo.latestId) < 0) return -1; if (tEncodeI64(pEncoder, ps->checkpointInfo.latestVer) < 0) return -1; if (tEncodeI64(pEncoder, ps->checkpointInfo.latestTime) < 0) return -1; + if (tEncodeI64(pEncoder, ps->checkpointInfo.latestSize) < 0) return -1; + if (tEncodeI8(pEncoder, ps->checkpointInfo.remoteBackup) < 0) return -1; if (tEncodeI64(pEncoder, ps->startTime) < 0) return -1; if (tEncodeI64(pEncoder, ps->startCheckpointId) < 0) return -1; if (tEncodeI64(pEncoder, ps->startCheckpointVer) < 0) return -1; @@ -381,6 +387,10 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI32(pDecoder, &entry.nodeId) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputQUsed) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputRate) < 0) return -1; + if (tDecodeDouble(pDecoder, &entry.procsTotal) < 0) return -1; + if (tDecodeDouble(pDecoder, &entry.procsThroughput) < 0) return -1; + if (tDecodeDouble(pDecoder, &entry.outputTotal) < 0) return -1; + if (tDecodeDouble(pDecoder, &entry.outputThroughput) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.sinkQuota) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.sinkDataSize) < 0) return -1; if (tDecodeI64(pDecoder, &entry.processedVer) < 0) return -1; @@ -393,6 +403,8 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestId) < 0) return -1; if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestVer) < 0) return -1; if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestTime) < 0) return -1; + if (tDecodeI64(pDecoder, &entry.checkpointInfo.latestSize) < 0) return -1; + if (tDecodeI8(pDecoder, &entry.checkpointInfo.remoteBackup) < 0) return -1; if (tDecodeI64(pDecoder, &entry.startTime) < 0) return -1; if (tDecodeI64(pDecoder, &entry.startCheckpointId) < 0) return -1; if (tDecodeI64(pDecoder, &entry.startCheckpointVer) < 0) return -1; @@ -469,7 +481,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { int32_t epSz = taosArrayGetSize(pTask->upstreamInfo.pList); if (tEncodeI32(pEncoder, epSz) < 0) return -1; for (int32_t i = 0; i < epSz; i++) { - SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); + SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1; } @@ -493,7 +505,7 @@ int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tSerializeSUseDbRspImp(pEncoder, &pTask->outputInfo.shuffleDispatcher.dbInfo) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; } - if (tEncodeI64(pEncoder, pTask->info.triggerParam) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->info.delaySchedParam) < 0) return -1; if (tEncodeI8(pEncoder, pTask->subtableWithoutMd5) < 0) return -1; if (tEncodeCStrWithLen(pEncoder, pTask->reserve, sizeof(pTask->reserve) - 1) < 0) return -1; @@ -545,7 +557,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { pTask->upstreamInfo.pList = taosArrayInit(epSz, POINTER_BYTES); for (int32_t i = 0; i < epSz; i++) { - SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo)); + SStreamUpstreamEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamUpstreamEpInfo)); if (pInfo == NULL) return -1; if (tDecodeStreamEpInfo(pDecoder, pInfo) < 0) { taosMemoryFreeClear(pInfo); @@ -576,7 +588,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDeserializeSUseDbRspImp(pDecoder, &pTask->outputInfo.shuffleDispatcher.dbInfo) < 0) return -1; if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; } - if (tDecodeI64(pDecoder, &pTask->info.triggerParam) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->info.delaySchedParam) < 0) return -1; if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER){ if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 83de642e51..fb745f86cb 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -759,7 +759,7 @@ void* getRowStateBuff(SStreamFileState* pFileState) { return pFileState->rowStat void* getStateFileStore(SStreamFileState* pFileState) { return pFileState->pFileStore; } bool isDeteled(SStreamFileState* pFileState, TSKEY ts) { - return pFileState->deleteMark > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); + return pFileState->deleteMark != INT64_MAX && pFileState->maxTs > 0 && ts < (pFileState->maxTs - pFileState->deleteMark); } bool isFlushedState(SStreamFileState* pFileState, TSKEY ts, TSKEY gap) { return ts <= (pFileState->flushMark + gap); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 3543ed574c..796a45d997 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -874,6 +874,17 @@ int32_t syncLogReplRecover(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEn sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64, pNode->vgId, terrstr(), index + 1); return -1; } + + if (pMsg->matchIndex == -1) { + // first time to restore + sInfo("vgId:%d, first time to restore sync log repl. peer: dnode:%d (%" PRIx64 "), repl-mgr:[%" PRId64 " %" PRId64 + ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 "), index:%" PRId64 + ", firstVer:%" PRId64 ", term:%" PRId64 ", lastMatchTerm:%" PRId64, + pNode->vgId, DID(&destId), destId.addr, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, + pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex, index, firstVer, term, + pMsg->lastMatchTerm); + } + if ((index + 1 < firstVer) || (term < 0) || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index f2b45c5b84..e7c4573c14 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -177,11 +177,10 @@ int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { continue; } - return 0; + return (terrno = 0); } - terrno = TSDB_CODE_FS_NO_VALID_DISK; - return -1; + return (terrno = TSDB_CODE_FS_NO_VALID_DISK); } const char *tfsGetPrimaryPath(STfs *pTfs) { return TFS_PRIMARY_DISK(pTfs)->path; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8969d98228..7bd746837f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -203,7 +203,7 @@ static FORCE_INLINE void cliMayUpdateFqdnCache(SHashObj* cache, char* dst); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn -static void cliHandleExcept(SCliConn* conn); +static void cliHandleExcept(SCliConn* conn, int32_t code); static void cliReleaseUnfinishedMsg(SCliConn* conn); static void cliHandleFastFail(SCliConn* pConn, int status); @@ -351,6 +351,7 @@ bool cliConnSendSeqMsg(int64_t refId, SCliConn* conn) { taosWLockLatch(&exh->latch); if (exh->handle == NULL) exh->handle = conn; exh->inited = 1; + exh->pThrd = conn->hostThrd; if (!QUEUE_IS_EMPTY(&exh->q)) { queue* h = QUEUE_HEAD(&exh->q); QUEUE_REMOVE(h); @@ -574,8 +575,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } -void cliHandleExcept(SCliConn* conn) { +void cliHandleExcept(SCliConn* conn, int32_t code) { tTrace("%s conn %p except ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); + if (code != TSDB_CODE_RPC_FQDN_ERROR) { + code = -1; + } cliHandleExceptImpl(conn, -1); } @@ -869,7 +873,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { while (transReadComplete(pBuf)) { tTrace("%s conn %p read complete", CONN_GET_INST_LABEL(conn), conn); if (pBuf->invalid) { - cliHandleExcept(conn); + cliHandleExcept(conn, -1); break; } else { cliHandleResp(conn); @@ -889,7 +893,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { tDebug("%s conn %p read error:%s, ref:%d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), T_REF_VAL_GET(conn)); conn->broken = true; - cliHandleExcept(conn); + cliHandleExcept(conn, -1); } } @@ -1040,7 +1044,7 @@ static void cliSendCb(uv_write_t* req, int status) { } else { if (!uv_is_closing((uv_handle_t*)&pConn->stream)) { tError("%s conn %p failed to write:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_err_name(status)); - cliHandleExcept(pConn); + cliHandleExcept(pConn, -1); } return; } @@ -1116,7 +1120,7 @@ void cliSend(SCliConn* pConn) { if (transQueueEmpty(&pConn->cliMsgs)) { tError("%s conn %p not msg to send", pTransInst->label, pConn); - cliHandleExcept(pConn); + cliHandleExcept(pConn, -1); return; } @@ -1189,7 +1193,7 @@ void cliSend(SCliConn* pConn) { if (status != 0) { tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), uv_err_name(status)); - cliHandleExcept(pConn); + cliHandleExcept(pConn, -1); } return; _RETURN: @@ -1245,7 +1249,8 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; - cliHandleFastFail(conn, -1); + cliHandleFastFail(conn, terrno); + terrno = 0; return; } struct sockaddr_in addr; @@ -1307,7 +1312,7 @@ static void cliSendBatchCb(uv_write_t* req, int status) { tDebug("%s conn %p failed to send batch msg, batch size:%d, msgLen:%d, reason:%s", CONN_GET_INST_LABEL(conn), conn, p->wLen, p->batchSize, uv_err_name(status)); - if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn); + if (!uv_is_closing((uv_handle_t*)&conn->stream)) cliHandleExcept(conn, -1); cliHandleBatchReq(nxtBatch, thrd); } else { @@ -1365,7 +1370,7 @@ static void cliHandleFastFail(SCliConn* pConn, int status) { cliDestroyBatch(pConn->pBatch); pConn->pBatch = NULL; } - cliHandleExcept(pConn); + cliHandleExcept(pConn, status); } void cliConnCb(uv_connect_t* req, int status) { @@ -1662,7 +1667,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { taosArrayPush(pThrd->timerList, &conn->timer); conn->timer = NULL; - cliHandleExcept(conn); + cliHandleExcept(conn, terrno); + terrno = 0; return; } @@ -1676,20 +1682,20 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { if (fd == -1) { tGError("%s conn %p failed to create socket, reason:%s", transLabel(pTransInst), conn, tstrerror(TAOS_SYSTEM_ERROR(errno))); - cliHandleExcept(conn); + cliHandleExcept(conn, -1); errno = 0; return; } int ret = uv_tcp_open((uv_tcp_t*)conn->stream, fd); if (ret != 0) { tGError("%s conn %p failed to set stream, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret)); - cliHandleExcept(conn); + cliHandleExcept(conn, -1); return; } ret = transSetConnOption((uv_tcp_t*)conn->stream, tsKeepAliveIdle); if (ret != 0) { tGError("%s conn %p failed to set socket opt, reason:%s", transLabel(pTransInst), conn, uv_err_name(ret)); - cliHandleExcept(conn); + cliHandleExcept(conn, -1); return; } @@ -2518,7 +2524,7 @@ static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t if (exh == NULL) { return NULL; } - + taosWLockLatch(&exh->latch); if (exh->pThrd == NULL && trans != NULL) { int idx = cliRBChoseIdx(trans); if (idx < 0) return NULL; @@ -2526,7 +2532,9 @@ static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t } pThrd = exh->pThrd; + taosWUnLockLatch(&exh->latch); transReleaseExHandle(transGetRefMgt(), handle); + return pThrd; } SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 08162ef158..ac33b1d6b7 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -20,7 +20,6 @@ #include "tutil.h" #include "walInt.h" - bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) { return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver; } @@ -154,7 +153,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { // validate body int32_t cryptedBodyLen = logContent->head.bodyLen; - if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){ + if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); } recordLen = walCkHeadSz + cryptedBodyLen; @@ -226,7 +225,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { goto _err; } - if (taosFsyncFile(pFile) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pFile) < 0) { wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -626,7 +625,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int32_t plainBodyLen = ckHead.head.bodyLen; int32_t cryptedBodyLen = plainBodyLen; - if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){ + if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { cryptedBodyLen = ENCRYPTED_LEN(cryptedBodyLen); } idxEntry.offset += sizeof(SWalCkHead) + cryptedBodyLen; @@ -636,7 +635,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { pWal->cfg.vgId, terrstr(), idxEntry.ver, idxEntry.offset, fLogNameStr); goto _err; } - if (taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pIdxFile, &idxEntry, sizeof(SWalIdxEntry)) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, failed to append file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); goto _err; @@ -644,7 +643,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { count++; } - if (taosFsyncFile(pIdxFile) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pIdxFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, faild to fsync file since %s. file:%s", pWal->cfg.vgId, terrstr(), fnameStr); goto _err; @@ -880,13 +879,13 @@ int walSaveMeta(SWal* pWal) { int n; // fsync the idx and log file at first to ensure validity of meta - if (taosFsyncFile(pWal->pIdxFile) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pIdxFile) < 0) { wError("vgId:%d, failed to sync idx file due to %s", pWal->cfg.vgId, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosFsyncFile(pWal->pLogFile) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pWal->pLogFile) < 0) { wError("vgId:%d, failed to sync log file due to %s", pWal->cfg.vgId, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -901,7 +900,8 @@ int walSaveMeta(SWal* pWal) { return -1; } - TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); + TdFilePtr pMetaFile = + taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_WRITE_THROUGH); if (pMetaFile == NULL) { wError("vgId:%d, failed to open file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); @@ -910,13 +910,13 @@ int walSaveMeta(SWal* pWal) { char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(pMetaFile, serialized, len)) { + if (pWal->cfg.level != TAOS_WAL_SKIP && len != taosWriteFile(pMetaFile, serialized, len)) { wError("vgId:%d, failed to write file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (taosFsyncFile(pMetaFile) < 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && taosFsyncFile(pMetaFile) < 0) { wError("vgId:%d, failed to sync file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), tmpFnameStr); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index cbfd0ef741..0e452a937b 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -79,8 +79,7 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; if (pWal->pLogFile != NULL) { - code = taosFsyncFile(pWal->pLogFile); - if (code != 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -91,8 +90,7 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { } } if (pWal->pIdxFile != NULL) { - code = taosFsyncFile(pWal->pIdxFile); - if (code != 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 25fec4a998..19345e0644 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "crypt.h" #include "os.h" #include "taoserror.h" #include "tchecksum.h" #include "tglobal.h" #include "walInt.h" -#include "crypt.h" int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { taosThreadMutexLock(&pWal->mutex); @@ -295,9 +295,9 @@ int32_t walEndSnapshot(SWal *pWal) { ver = TMAX(ver - pWal->vers.logRetention, pWal->vers.firstVer - 1); // compatible mode for refVer - bool hasTopic = false; + bool hasTopic = false; int64_t refVer = INT64_MAX; - void *pIter = NULL; + void *pIter = NULL; while (1) { pIter = taosHashIterate(pWal->pRefHash, pIter); if (pIter == NULL) break; @@ -396,8 +396,7 @@ int32_t walRollImpl(SWal *pWal) { int32_t code = 0; if (pWal->pIdxFile != NULL) { - code = taosFsyncFile(pWal->pIdxFile); - if (code != 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pIdxFile)) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); goto END; } @@ -409,8 +408,7 @@ int32_t walRollImpl(SWal *pWal) { } if (pWal->pLogFile != NULL) { - code = taosFsyncFile(pWal->pLogFile); - if (code != 0) { + if (pWal->cfg.level != TAOS_WAL_SKIP && (code = taosFsyncFile(pWal->pLogFile)) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); goto END; } @@ -510,12 +508,15 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy wDebug("vgId:%d, wal write log %" PRId64 ", msgType: %s, cksum head %u cksum body %u", pWal->cfg.vgId, index, TMSG_INFO(msgType), pWal->writeHead.cksumHead, pWal->writeHead.cksumBody); - code = walWriteIndex(pWal, index, offset); - if (code < 0) { - goto END; + if (pWal->cfg.level != TAOS_WAL_SKIP) { + code = walWriteIndex(pWal, index, offset); + if (code < 0) { + goto END; + } } - if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { + if (pWal->cfg.level != TAOS_WAL_SKIP && + taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -524,17 +525,17 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } int32_t cyptedBodyLen = plainBodyLen; - char* buf = (char*)body; - char* newBody = NULL; - char* newBodyEncrypted = NULL; + char *buf = (char *)body; + char *newBody = NULL; + char *newBodyEncrypted = NULL; - if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){ + if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { cyptedBodyLen = ENCRYPTED_LEN(cyptedBodyLen); - + newBody = taosMemoryMalloc(cyptedBodyLen); - if(newBody == NULL){ + if (newBody == NULL) { wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(errno)); code = -1; goto END; } @@ -542,11 +543,11 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy memcpy(newBody, body, plainBodyLen); newBodyEncrypted = taosMemoryMalloc(cyptedBodyLen); - if(newBodyEncrypted == NULL){ + if (newBodyEncrypted == NULL) { wError("vgId:%d, file:%" PRId64 ".log, failed to malloc since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), - strerror(errno)); + strerror(errno)); code = -1; - if(newBody != NULL) taosMemoryFreeClear(newBody); + if (newBody != NULL) taosMemoryFreeClear(newBody); goto END; } @@ -559,29 +560,29 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy int32_t count = CBC_Encrypt(&opts); - //wDebug("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", - // pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__); + // wDebug("vgId:%d, file:%" PRId64 ".log, index:%" PRId64 ", CBC_Encrypt cryptedBodyLen:%d, plainBodyLen:%d, %s", + // pWal->cfg.vgId, walGetLastFileFirstVer(pWal), index, count, plainBodyLen, __FUNCTION__); buf = newBodyEncrypted; } - - if (taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { + + if (pWal->cfg.level != TAOS_WAL_SKIP && taosWriteFile(pWal->pLogFile, (char *)buf, cyptedBodyLen) != cyptedBodyLen) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); code = -1; - if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){ + if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { taosMemoryFreeClear(newBody); taosMemoryFreeClear(newBodyEncrypted); } goto END; } - if(pWal->cfg.encryptAlgorithm == DND_CA_SM4){ + if (pWal->cfg.encryptAlgorithm == DND_CA_SM4) { taosMemoryFreeClear(newBody); - taosMemoryFreeClear(newBodyEncrypted); - //wInfo("vgId:%d, free newBody newBodyEncrypted %s", - // pWal->cfg.vgId, __FUNCTION__); + taosMemoryFreeClear(newBodyEncrypted); + // wInfo("vgId:%d, free newBody newBodyEncrypted %s", + // pWal->cfg.vgId, __FUNCTION__); } // set status @@ -693,6 +694,10 @@ int32_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in } void walFsync(SWal *pWal, bool forceFsync) { + if (pWal->cfg.level == TAOS_WAL_SKIP) { + return; + } + taosThreadMutexLock(&pWal->mutex); if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5a3dc867e6..4635ec340d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -2741,7 +2741,7 @@ int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t nBuf) { DEFINE_VAR(cmprAlg) - if (lvl != 0 && lossyFloat) { + if (l2 == L2_TSZ && lvl != 0 && lossyFloat) { return tsCompressFloatLossyImp(pIn, nEle, pOut); } FUNC_COMPRESS_IMPL(pIn, nIn, nEle, pOut, nOut, cmprAlg, pBuf, nBuf, TSDB_DATA_TYPE_FLOAT, 1); @@ -2760,7 +2760,7 @@ int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf, int32_t nBuf) { DEFINE_VAR(cmprAlg) - if (lvl != 0 && lossyDouble) { + if (l2 == L2_TSZ && lvl != 0 && lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(pIn, nEle, pOut); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ad811cc891..0f594af0e9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -259,7 +259,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED, "The database encryption function grant expired") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") @@ -499,9 +498,20 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LACK_OF_BASIC, "Lack of basic functio TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJ_NOT_EXIST, "Grant object not exist") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND, "The historial active code does not match") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mismatch with active code") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expiration time of optional grant item is too large") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of views has reached the licensed upper limit") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_BASIC_EXPIRED, "License expired for basic functions") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_EXPIRED, "License expired for stream function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SUBSCRIPTION_EXPIRED, "License expired for subscription function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_EXPIRED, "License expired for view function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_EXPIRED, "License expired for audit function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_EXPIRED, "License expired for CSV function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MULTI_STORAGE_EXPIRED, "License expired for multi-tier storage function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OBJECT_STROAGE_EXPIRED, "License expired for object storage function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUAL_REPLICA_HA_EXPIRED,"License expired for dual-replica HA function") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_ENCRYPTION_EXPIRED, "License expired for database encryption function") + // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") diff --git a/source/util/src/version.c.in b/source/util/src/version.c.in index ec6449a02f..2446bf9278 100644 --- a/source/util/src/version.c.in +++ b/source/util/src/version.c.in @@ -2,6 +2,6 @@ char version[64] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; -char buildinfo[64] = "Built ${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} at ${TD_VER_DATE}"; +char buildinfo[64] = "${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}"; void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/tests/army/alter/alterConfig.py b/tests/army/alter/alterConfig.py new file mode 100644 index 0000000000..7413e5e5a6 --- /dev/null +++ b/tests/army/alter/alterConfig.py @@ -0,0 +1,55 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * + + +class TDTestCase(TBase): + def alterSupportVnodes(self): + tdLog.info(f"test function of altering supportVnodes") + + tdSql.execute("alter dnode 1 'supportVnodes' '128'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "128") + + tdSql.execute("alter dnode 1 'supportVnodes' '64'") + time.sleep(1) + tdSql.query('show dnodes') + tdSql.checkData(0, 3, "64") + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # TS-4721 + self.alterSupportVnodes() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/cluster/incSnapshot.py b/tests/army/cluster/incSnapshot.py new file mode 100644 index 0000000000..dfd8d95c9c --- /dev/null +++ b/tests/army/cluster/incSnapshot.py @@ -0,0 +1,110 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + updatecfgDict = { + 'slowLogScope':"query" + } + + def init(self, conn, logSql, replicaVar=3): + super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1") + self.valgrind = 0 + self.childtable_count = 10 + # tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 3) + tdSql.execute(f"use {self.db}") + autoGen.create_stable(self.stb, 5, 10, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1000) + tdSql.execute(f"flush database {self.db}") + sc.dnodeStop(3) + # clusterDnodes.stoptaosd(1) + # clusterDnodes.starttaosd(3) + # time.sleep(5) + # clusterDnodes.stoptaosd(2) + # clusterDnodes.starttaosd(1) + # time.sleep(5) + autoGen.insert_data(5000, True) + self.flushDb(True) + # wait flush operation over + time.sleep(5) + + # sql = 'show vnodes;' + # while True: + # bFinish = True + # param_list = tdSql.query(sql, row_tag=True) + # for param in param_list: + # if param[3] == 'leading' or param[3] == 'following': + # bFinish = False + # break + # if bFinish: + # break + self.snapshotAgg() + time.sleep(10) + sc.dnodeStopAll() + for i in range(1, 4): + path = clusterDnodes.getDnodeDir(i) + dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + dirs = glob.glob(dnodesRootDir) + for dir in dirs: + if os.path.isdir(dir): + self.remove_directory(os.path.join(dir, "wal")) + + sc.dnodeStart(1) + sc.dnodeStart(2) + sc.dnodeStart(3) + sql = "show vnodes;" + time.sleep(10) + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'offline': + tdLog.exit( + "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + + self.timestamp_step = 1000 + self.insert_rows = 6000 + self.checkInsertCorrect() + self.checkAggCorrect() + + def remove_directory(self, directory): + try: + shutil.rmtree(directory) + tdLog.debug("delete dir: %s " % (directory)) + except OSError as e: + tdLog.exit("delete fail dir: %s " % (directory)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/community/cluster/snapshot.json b/tests/army/cluster/snapshot.json similarity index 100% rename from tests/army/community/cluster/snapshot.json rename to tests/army/cluster/snapshot.json diff --git a/tests/army/community/cluster/snapshot.py b/tests/army/cluster/snapshot.py similarity index 100% rename from tests/army/community/cluster/snapshot.py rename to tests/army/cluster/snapshot.py diff --git a/tests/army/community/cluster/splitVgroupByLearner.json b/tests/army/cluster/splitVgroupByLearner.json similarity index 100% rename from tests/army/community/cluster/splitVgroupByLearner.json rename to tests/army/cluster/splitVgroupByLearner.json diff --git a/tests/army/community/cluster/splitVgroupByLearner.py b/tests/army/cluster/splitVgroupByLearner.py similarity index 100% rename from tests/army/community/cluster/splitVgroupByLearner.py rename to tests/army/cluster/splitVgroupByLearner.py diff --git a/tests/army/community/cmdline/fullopt.py b/tests/army/cmdline/fullopt.py similarity index 100% rename from tests/army/community/cmdline/fullopt.py rename to tests/army/cmdline/fullopt.py diff --git a/tests/army/db-encrypt/basic.py b/tests/army/db-encrypt/basic.py new file mode 100644 index 0000000000..8d30bbcfe2 --- /dev/null +++ b/tests/army/db-encrypt/basic.py @@ -0,0 +1,63 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.srvCtl import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +# from frame.server.dnodes import * +# from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=1): + super(TDTestCase, self).init(conn, logSql, replicaVar=1, checkColName="c1") + self.valgrind = 0 + self.db = "test" + self.stb = "meters" + self.childtable_count = 10 + tdSql.init(conn.cursor(), logSql) + + def create_encrypt_db(self): + + tdSql.execute("create encrypt_key '1234567890'") + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 1, "ENCRYPT_ALGORITHM 'sm4'") + tdSql.execute(f"use {self.db}") + autoGen.create_stable(self.stb, 2, 3, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1000) + + tdSql.query(f"select * from {self.db}.{self.stb}") + tdSql.checkRows(1000 * self.childtable_count) + + self.timestamp_step = 1000 + self.insert_rows = 1000 + + self.checkInsertCorrect() + + def create_encrypt_db_error(self): + tdSql.error("create encrypt_key '123'") + tdSql.error("create encrypt_key '12345678abcdefghi'") + tdSql.error("create database test ENCRYPT_ALGORITHM 'sm4'") + + def run(self): + self.create_encrypt_db_error() + self.create_encrypt_db() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/etime.py b/tests/army/frame/etime.py index 2ee223e122..808ebaf3b0 100644 --- a/tests/army/frame/etime.py +++ b/tests/army/frame/etime.py @@ -19,5 +19,3 @@ import sys import os import time import datetime - - diff --git a/tests/army/frame/eutil.py b/tests/army/frame/eutil.py new file mode 100644 index 0000000000..a32e6e9a38 --- /dev/null +++ b/tests/army/frame/eutil.py @@ -0,0 +1,28 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about tools funciton extension +# + +import sys +import os +import time +import datetime +import psutil + + +# cpu frequent as random +def cpuRand(max): + decimal = int(str(psutil.cpu_freq().current).split(".")[1]) + return decimal % max \ No newline at end of file diff --git a/tests/army/community/insert/insert_basic.py b/tests/army/insert/insert_basic.py similarity index 61% rename from tests/army/community/insert/insert_basic.py rename to tests/army/insert/insert_basic.py index 481db4eadd..1f2488a756 100644 --- a/tests/army/community/insert/insert_basic.py +++ b/tests/army/insert/insert_basic.py @@ -35,18 +35,18 @@ class TDTestCase(TBase): tdSql.execute("create database db_geometry;") tdSql.execute("use db_geometry;") tdSql.execute("create table t_ge (ts timestamp, id int, c1 GEOMETRY(512));") - tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT ((0 0), (1 1))');") - tdSql.execute("insert into t_ge values(now, 1, 'MULTIPOINT (0 0, 1 1)');") - tdSql.execute("insert into t_ge values(now, 2, 'POINT (0 0)');") - tdSql.execute("insert into t_ge values(now, 2, 'POINT EMPTY');") - tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING (0 0, 0 1, 1 2)');") - tdSql.execute("insert into t_ge values(now, 3, 'LINESTRING EMPTY');") - tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))');") - tdSql.execute("insert into t_ge values(now, 4, 'POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))');") - tdSql.execute("insert into t_ge values(now, 4, 'POLYGON EMPTY');") - tdSql.execute("insert into t_ge values(now, 5, 'MULTILINESTRING ((0 0, 1 1), (2 2, 3 3))');") - tdSql.execute("insert into t_ge values(now, 6, 'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((4 3, 6 3, 6 1, 4 1, 4 3)))');") - tdSql.execute("insert into t_ge values(now, 7, 'GEOMETRYCOLLECTION (MULTIPOINT((0 0), (1 1)), POINT(3 4), LINESTRING(2 3, 3 4))');") + tdSql.execute("insert into t_ge values(1717122943000, 1, 'MULTIPOINT ((0 0), (1 1))');") + tdSql.execute("insert into t_ge values(1717122944000, 1, 'MULTIPOINT (0 0, 1 1)');") + tdSql.execute("insert into t_ge values(1717122945000, 2, 'POINT (0 0)');") + tdSql.execute("insert into t_ge values(1717122946000, 2, 'POINT EMPTY');") + tdSql.execute("insert into t_ge values(1717122947000, 3, 'LINESTRING (0 0, 0 1, 1 2)');") + tdSql.execute("insert into t_ge values(1717122948000, 3, 'LINESTRING EMPTY');") + tdSql.execute("insert into t_ge values(1717122949000, 4, 'POLYGON ((0 0, 1 0, 1 1, 0 1, 0 0))');") + tdSql.execute("insert into t_ge values(1717122950000, 4, 'POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0), (1 1, 1 2, 2 2, 2 1, 1 1))');") + tdSql.execute("insert into t_ge values(1717122951000, 4, 'POLYGON EMPTY');") + tdSql.execute("insert into t_ge values(1717122952000, 5, 'MULTILINESTRING ((0 0, 1 1), (2 2, 3 3))');") + tdSql.execute("insert into t_ge values(1717122953000, 6, 'MULTIPOLYGON (((1 1, 1 3, 3 3, 3 1, 1 1)), ((4 3, 6 3, 6 1, 4 1, 4 3)))');") + tdSql.execute("insert into t_ge values(1717122954000, 7, 'GEOMETRYCOLLECTION (MULTIPOINT((0 0), (1 1)), POINT(3 4), LINESTRING(2 3, 3 4))');") tdSql.query("select * from t_ge;") tdSql.checkRows(12) tdSql.query("select * from t_ge where id=1;") diff --git a/tests/army/community/insert/test_column_tag_boundary.py b/tests/army/insert/test_column_tag_boundary.py similarity index 91% rename from tests/army/community/insert/test_column_tag_boundary.py rename to tests/army/insert/test_column_tag_boundary.py index 4c04fd3f9b..4f1245d728 100644 --- a/tests/army/community/insert/test_column_tag_boundary.py +++ b/tests/army/insert/test_column_tag_boundary.py @@ -22,6 +22,7 @@ class TDTestCase(TBase): self.child_table_num = 1 self.insert_round_num = 700 self.row_num_per_round = 15 + self.start_ts = 1704082431000 def prepare_data(self): # database @@ -39,7 +40,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_binary%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'a' * self.max_column_length) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'a' * self.max_column_length) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_binary{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -63,7 +64,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_varchar%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), 'b' * self.max_column_length) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), 'b' * self.max_column_length) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_varchar{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -98,7 +99,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_nchar%s values" % (i+1) for k in range(self.row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), column) tdSql.execute(sql) tdLog.info(f"Insert {self.row_num_per_round} rows data into ct_nchar{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -124,7 +125,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_varbinary%s values" % (i+1) for k in range(row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column) tdSql.execute(sql) tdLog.info(f"Insert {row_num_per_round} rows data into ct_varbinary{i+1} {j+1} times successfully") tdSql.execute("flush database db;") @@ -153,7 +154,7 @@ class TDTestCase(TBase): for j in range(self.insert_round_num): sql = "insert into ct_json_tag%s values" % (i+1) for k in range(row_num_per_round): - sql += "(now+%ss, '%s')," % (str(j * 10 + k + 1), '\\x' + column) + sql += "(%s, '%s')," % (str(self.start_ts + (j * self.insert_round_num + k * self.row_num_per_round + 1)), '\\x' + column) tdSql.execute(sql) tdLog.info(f"Insert {row_num_per_round} rows data into ct_json_tag{i+1} {j+1} times successfully") tdSql.execute("flush database db;") diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/multi-level/mlevel_basic.json similarity index 100% rename from tests/army/enterprise/multi-level/mlevel_basic.json rename to tests/army/multi-level/mlevel_basic.json diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/multi-level/mlevel_basic.py similarity index 100% rename from tests/army/enterprise/multi-level/mlevel_basic.py rename to tests/army/multi-level/mlevel_basic.py diff --git a/tests/army/query/accuracy/test_query_accuracy.py b/tests/army/query/accuracy/test_query_accuracy.py new file mode 100644 index 0000000000..f7b9c00a28 --- /dev/null +++ b/tests/army/query/accuracy/test_query_accuracy.py @@ -0,0 +1,77 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * +import random +import string + + +class TDTestCase(TBase): + """Add test case to verify the complicated query accuracy + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def prepare_data(self): + # database for case TS-4806 + tdSql.execute("create database db_ts4806;") + tdSql.execute("use db_ts4806;") + # super table + tdSql.execute("create table st (ts timestamp, adl float, bdl float, cdl float, ady float, bdy float, cdy float) \ + tags(pt_radio float, ct_ratio float, rated_cap float, ta_id varchar(128), id varchar(128), area_code \ + varchar(128), zdy_flag int, elec_cust_name bigint,bureau_code bigint, fl_name varchar(32), classify_id \ + varchar(128));") + # child table + tdSql.execute("create table ct_1 using st tags(1.2, 1.3, 3.4, '271000276', '30000001', '10001', 1, 10001, 2000001, 'beijing', '13169');") + tdSql.execute("create table ct_2 using st tags(2.1, 1.2, 3.3, '271000277', '30000002', '10002', 1, 10002, 2000002, 'shanghai', '13141');") + tdSql.execute("create table ct_3 using st tags(3.1, 4.2, 5.3, '271000278', '30000003', '10003', 0, 10003, 2000003, 'guangzhou', '13151');") + # insert data for ts4806 + start_ts = 1705783972000 + data = [ + (1.1, 2.2, 3.3, 1.1, 2.2, 3.3), + (1.2, 2.3, 3.4, 1.2, 2.3, 3.4), + (1.3, 2.4, 3.5, 1.3, 2.4, 3.5), + (1.4, 2.5, 3.6, 1.4, 2.5, 3.6), + (1.5, 2.6, 3.7, 1.5, 2.6, 3.7), + (1.6, 2.7, 3.8, 1.6, 2.7, 3.8), + (1.7, 2.8, 3.9, 1.7, 2.8, 3.9), + (1.8, 2.9, 4.0, 1.8, 2.9, 4.0), + (1.9, 4.2, 4.1, 1.9, 3.0, 4.1), + (1.2, 3.1, 4.2, 2.0, 3.1, 4.2) + ] + index = [1, 2, 5, 0, 7, 3, 8, 4, 6, 9] + for ct in ['ct_1', 'ct_2']: + for i in range(10): + sql = f"insert into {ct} values" + for j in range(1000): + sql += f"({start_ts + i * 1000 * 1000 + j * 1000}, {','.join([str(item) for item in data[index[i]]])})," + sql += ";" + tdSql.execute(sql) + + def test_ts4806(self): + tdSql.execute("use db_ts4806;") + tdSql.query("select _wstart, cj.id, count(*) from st cj where cj.ts >= '2024-01-21 04:52:52.000' and cj.ts <= ' 2024-01-21 07:39:31.000' \ + and cj.zdy_flag = 1 and cj.id in ('30000001', '30000002') partition by cj.id event_window start with \ + (CASE WHEN cj.adl >= cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl \ + THEN cj.bdl ELSE cj.cdl END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap > 1 end with (CASE WHEN cj.adl >= \ + cj.bdl AND cj.adl >= cj.cdl THEN cj.adl WHEN cj.bdl >= cj.adl AND cj.bdl >= cj.cdl THEN cj.bdl ELSE cj.cdl \ + END) * cj.ct_ratio * 0.4 * 1.732 / cj.rated_cap <= 1 HAVING count(*) >= 4 order by _wstart, cj.id;") + tdSql.checkRows(5) + tdSql.checkData(4, 1, '30000002') + tdSql.checkData(4, 2, 1001) + + def run(self): + self.prepare_data() + self.test_ts4806() + + def stop(self): + tdSql.execute("drop database db_ts4806;") + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/army/community/query/cquery_basic.json b/tests/army/query/cquery_basic.json similarity index 100% rename from tests/army/community/query/cquery_basic.json rename to tests/army/query/cquery_basic.json diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/query/fill/fill_desc.py similarity index 100% rename from tests/army/community/query/fill/fill_desc.py rename to tests/army/query/fill/fill_desc.py diff --git a/tests/army/community/query/function/test_func_elapsed.py b/tests/army/query/function/test_func_elapsed.py similarity index 100% rename from tests/army/community/query/function/test_func_elapsed.py rename to tests/army/query/function/test_func_elapsed.py diff --git a/tests/army/community/query/query_basic.json b/tests/army/query/query_basic.json similarity index 100% rename from tests/army/community/query/query_basic.json rename to tests/army/query/query_basic.json diff --git a/tests/army/community/query/query_basic.py b/tests/army/query/query_basic.py similarity index 100% rename from tests/army/community/query/query_basic.py rename to tests/army/query/query_basic.py diff --git a/tests/army/community/query/show.py b/tests/army/query/show.py similarity index 80% rename from tests/army/community/query/show.py rename to tests/army/query/show.py index 980e60387a..72b84954cd 100644 --- a/tests/army/community/query/show.py +++ b/tests/army/query/show.py @@ -118,6 +118,22 @@ class TDTestCase(TBase): tdSql.query(sql) tdSql.checkRows(5) + # verification for TD-30030 + tdSql.execute("create table t100 (ts timestamp, pk varchar(20) primary key, c1 varchar(100)) tags (id int)") + tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a1', '100')") + tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a2', '200')") + tdSql.execute("insert into ct1 using t100 tags(1) values('2024-05-17 14:58:52.902', 'a3', '300')") + tdSql.execute("insert into ct2 using t100 tags(2) values('2024-05-17 14:58:52.902', 'a2', '200')") + tdSql.execute("create view v100 as select * from t100") + tdSql.execute("create view v200 as select * from ct1") + + tdSql.error("show tags from v100", expectErrInfo="Tags can only applied to super table and child table") + tdSql.error("show tags from v200", expectErrInfo="Tags can only applied to super table and child table") + + tdSql.execute("create table t200 (ts timestamp, pk varchar(20) primary key, c1 varchar(100))") + + tdSql.error("show tags from t200", expectErrInfo="Tags can only applied to super table and child table") + def checkShow(self): # not support sql = "show accounts;" diff --git a/tests/army/query/subquery/subqueryBugs.py b/tests/army/query/subquery/subqueryBugs.py new file mode 100644 index 0000000000..8442474d40 --- /dev/null +++ b/tests/army/query/subquery/subqueryBugs.py @@ -0,0 +1,94 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * + + +class TDTestCase(TBase): + + def ts_30189(self): + tdLog.info("create database ts_30189") + tdSql.execute(f"create database ts_30189") + tdSql.execute(f"use ts_30189") + sqls = [ + "CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256))", + "CREATE TABLE demo_201000008 USING demo (deviceid) TAGS ('201000008')", + "CREATE TABLE demo_K201000258 USING demo (deviceid) TAGS ('K201000258')", + "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-11-30 23:59:27.255', 51412.900999999998021)", + "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:11:28.179', 51458.900999999998021)", + "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-04 23:12:28.180', 51458.800999999999476)", + "INSERT INTO demo_201000008 (_ts,faev) VALUES ('2023-12-31 23:59:36.108', 52855.400999999998021)", + "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-11-30 23:59:00.365', 258839.234375000000000)", + "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:00:00.381', 272188.843750000000000)", + "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-28 05:01:00.600', 13.909012794494629)", + "INSERT INTO demo_K201000258 (_ts,faev) VALUES ('2023-12-31 23:59:00.366', 1886.711303710937500)", + ] + tdSql.executes(sqls) + sql1 = ''' + SELECT ts, deviceid, faev FROM ( + ( + SELECT deviceid, ts, faev FROM ( + SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev + FROM demo + WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' + PARTITION BY deviceid + ) WHERE diff_faev < 0 + ) UNION ALL + ( + SELECT deviceid, ts, faev FROM ( + SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev + FROM ( SELECT deviceid, _ts as ts , faev FROM demo + WHERE deviceid in ('201000008') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' + ORDER BY ts desc ) PARTITION BY deviceid + ) WHERE diff_faev > 0 + ) + UNION ALL + ( + SELECT deviceid, LAST(_ts) AS ts, LAST(faev) AS faev FROM demo + WHERE deviceid in ('201000008') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' + PARTITION BY deviceid INTERVAL(1n) + ) + ) order by ts + ''' + tdSql.query(sql1) + tdSql.checkRows(4) + + row1 = ['2023-11-30 23:59:27.255', "201000008", 51412.900999999998021] + row2 = ['2023-12-04 23:11:28.179', "201000008", 51458.900999999998021] + row3 = ['2023-12-04 23:12:28.180', "201000008", 51458.800999999999476] + row4 = ['2023-12-31 23:59:36.108', "201000008", 52855.400999999998021] + + rows = [row1, row2, row3, row4] + tdSql.checkDataMem(sql1, rows) + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # TS-30189 + self.ts_30189() + + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/community/query/test_compare.py b/tests/army/query/test_compare.py similarity index 100% rename from tests/army/community/query/test_compare.py rename to tests/army/query/test_compare.py diff --git a/tests/army/community/query/test_join.py b/tests/army/query/test_join.py similarity index 100% rename from tests/army/community/query/test_join.py rename to tests/army/query/test_join.py diff --git a/tests/army/enterprise/s3/s3Basic.json b/tests/army/s3/s3Basic.json similarity index 100% rename from tests/army/enterprise/s3/s3Basic.json rename to tests/army/s3/s3Basic.json diff --git a/tests/army/enterprise/s3/s3Basic.py b/tests/army/s3/s3Basic.py similarity index 97% rename from tests/army/enterprise/s3/s3Basic.py rename to tests/army/s3/s3Basic.py index 9634b8edb0..f94fe611a6 100644 --- a/tests/army/enterprise/s3/s3Basic.py +++ b/tests/army/s3/s3Basic.py @@ -19,6 +19,7 @@ import taos import frame import frame.etool import frame.eos +import frame.eutil from frame.log import * from frame.cases import * @@ -46,17 +47,20 @@ for test: class TDTestCase(TBase): + index = eutil.cpuRand(20) + 1 + bucketName = f"ci-bucket{index}" updatecfgDict = { "supportVnodes":"1000", 's3EndPoint': 'http://192.168.1.52:9000', 's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', - 's3BucketName': 'ci-bucket', + 's3BucketName': f'{bucketName}', 's3PageCacheSize': '10240', "s3UploadDelaySec": "10", 's3MigrateIntervalSec': '600', 's3MigrateEnabled': '1' } + tdLog.info(f"assign bucketName is {bucketName}\n") maxFileSize = (128 + 10) * 1014 * 1024 # add 10M buffer def insertData(self): @@ -241,10 +245,9 @@ class TDTestCase(TBase): # def preDb(self, vgroups): - - cnt = int(time.time())%3 + 1 + cnt = int(time.time())%2 + 1 for i in range(cnt): - vg = int(time.time()*1000)%10 + 1 + vg = eutil.cpuRand(9) + 1 sql = f"create database predb vgroups {vg}" tdSql.execute(sql, show=True) sql = "drop database predb" diff --git a/tests/army/enterprise/s3/s3Basic1.json b/tests/army/s3/s3Basic1.json similarity index 100% rename from tests/army/enterprise/s3/s3Basic1.json rename to tests/army/s3/s3Basic1.json diff --git a/tests/army/community/storage/compressBasic.py b/tests/army/storage/compressBasic.py similarity index 84% rename from tests/army/community/storage/compressBasic.py rename to tests/army/storage/compressBasic.py index 48254fe3ac..64d0d4148f 100644 --- a/tests/army/community/storage/compressBasic.py +++ b/tests/army/storage/compressBasic.py @@ -48,6 +48,15 @@ class TDTestCase(TBase): "bigint","bigint unsigned","timestamp","bool","float","double","binary(16)","nchar(16)", "varchar(16)","varbinary(16)"] + # encode + encodes = [ + [["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8B"]], + [["timestamp","bigint","bigint unsigned"], ["Delta-i"]], + [["bool"], ["Bit-packing"]], + [["float","double"], ["Delta-d"]] + ] + + def combineValid(self, datatype, encode, compress): if datatype != "float" and datatype != "double": if compress == "tsz": @@ -55,13 +64,6 @@ class TDTestCase(TBase): return True def genAllSqls(self, stbName, max): - # encode - encodes = [ - [["tinyint","tinyint unsigned","smallint","smallint unsigned","int","int unsigned","bigint","bigint unsigned"], ["simple8B"]], - [["timestamp","bigint","bigint unsigned"], ["Delta-i"]], - [["bool"], ["Bit-packing"]], - [["float","double"], ["Delta-d"]] - ] c = 0 # column number t = 0 # table number @@ -70,7 +72,7 @@ class TDTestCase(TBase): sql = "" # loop append sqls - for lines in encodes: + for lines in self.encodes: for datatype in lines[0]: for encode in lines[1]: for compress in self.compresses: @@ -217,7 +219,34 @@ class TDTestCase(TBase): "alter table db.errstb modify column c3 compress 'xz';" ] tdSql.errors(sqls) - + + # add column + def checkAddColumn(self): + c = 0 + tbname = f"{self.db}.tbadd" + sql = f"create table {tbname}(ts timestamp, c0 int) tags(area int);" + tdSql.execute(sql) + + # loop append sqls + for lines in self.encodes: + for datatype in lines[0]: + for encode in lines[1]: + for compress in self.compresses: + for level in self.levels: + if self.combineValid(datatype, encode, compress): + sql = f"alter table {tbname} add column col{c} {datatype} ENCODE '{encode}' COMPRESS '{compress}' LEVEL '{level}';" + tdSql.execute(sql, 3, True) + c += 1 + + # alter error + sqls = [ + f"alter table {tbname} add column a1 int ENCODE 'simple8bAA';", + f"alter table {tbname} add column a2 int COMPRESS 'AABB';", + f"alter table {tbname} add column a3 bigint LEVEL 'high1';", + f"alter table {tbname} add column a4 BINARY(12) ENCODE 'simple8b' LEVEL 'high2';", + f"alter table {tbname} add column a5 VARCHAR(16) ENCODE 'simple8b' COMPRESS 'gzip' LEVEL 'high3';" + ] + tdSql.errors(sqls) def validCreate(self): sqls = self.genAllSqls(self.stb, 50) @@ -238,6 +267,9 @@ class TDTestCase(TBase): # check alter and write self.checkAlter() + # check add column + self.checkAddColumn() + def checkCorrect(self): # check data correct tbname = f"{self.db}.{self.stb}" diff --git a/tests/army/community/storage/oneStageComp.json b/tests/army/storage/oneStageComp.json similarity index 100% rename from tests/army/community/storage/oneStageComp.json rename to tests/army/storage/oneStageComp.json diff --git a/tests/army/community/storage/oneStageComp.py b/tests/army/storage/oneStageComp.py similarity index 100% rename from tests/army/community/storage/oneStageComp.py rename to tests/army/storage/oneStageComp.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7348b1fea0..1eaf17ccb1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,22 +10,26 @@ # # army-test # -,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 -,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py -,,y,army,./pytest.sh python3 ./test.py -f community/query/test_join.py -,,y,army,./pytest.sh python3 ./test.py -f community/query/test_compare.py -,,y,army,./pytest.sh python3 ./test.py -f community/insert/test_column_tag_boundary.py -,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3 -,,n,army,python3 ./test.py -f community/cmdline/fullopt.py -,,n,army,python3 ./test.py -f community/query/show.py -N 3 -,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1 -,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py +,,n,army,python3 ./test.py -f s3/s3Basic.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py +,,y,army,./pytest.sh python3 ./test.py -f query/test_join.py +,,y,army,./pytest.sh python3 ./test.py -f query/test_compare.py +,,y,army,./pytest.sh python3 ./test.py -f insert/test_column_tag_boundary.py +,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_desc.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py +,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3 +,,n,army,python3 ./test.py -f cmdline/fullopt.py +,,n,army,python3 ./test.py -f query/show.py -N 3 +,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3 +,,y,army,./pytest.sh python3 ./test.py -f storage/oneStageComp.py -N 3 -L 3 -D 1 +,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3 # # system test @@ -323,6 +327,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/subscribe_stream_privilege.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/empty_identifier.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_create.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/composite_primary_key_insert.py @@ -993,6 +998,7 @@ ,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py #tsim test +,,y,script,./test.sh -f tsim/query/timeline.sim ,,y,script,./test.sh -f tsim/join/join.sim ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ,,y,script,./test.sh -f tsim/parser/where.sim diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index ef9413a9d5..e58f890ccd 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -315,19 +315,22 @@ function run_thread() { fi if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" - local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" - # if [ $ent -ne 0 ]; then - # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" - # fi - mkdir "$build_dir" 2>/dev/null - if [ $? -eq 0 ]; then - # scp build binary - cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" - echo "$cmd" - $cmd >/dev/null - fi fi + # scp build binary and unit test log + local build_dir=$log_dir/build_${hosts[index]} + local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" + local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/" + + mkdir "$build_dir" 2>/dev/null + if [ $? -eq 0 ]; then + cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" + echo "$cmd" + $cmd >/dev/null + cmd="$scpcmd:${remote_unit_test_log_dir}/* ${build_dir}/" + echo "$cmd" + $cmd >/dev/null + fi + # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index df50e8031c..7bb2f42495 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -201,6 +201,9 @@ class TDCom: self.cast_tag_stb_filter_des_select_elm = "ts, t1, t2, t3, t4, cast(t1 as TINYINT UNSIGNED), t6, t7, t8, t9, t10, cast(t2 as varchar(256)), t12, cast(t3 as bool)" self.tag_count = len(self.tag_filter_des_select_elm.split(",")) self.state_window_range = list() + + self.custom_col_val = 0 + self.part_val_list = [1, 2] # def init(self, conn, logSql): # # tdSql.init(conn.cursor(), logSql) @@ -1259,7 +1262,7 @@ class TDCom: default_ctbname_index_start_num += 1 tdSql.execute(create_stable_sql) - def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None): + def sgen_column_value_list(self, column_elm_list, need_null, ts_value=None, additional_ts=None, custom_col_index=None, col_value_type=None, force_pk_val=None): """_summary_ Args: @@ -1269,6 +1272,8 @@ class TDCom: """ self.column_value_list = list() self.ts_value = self.genTs()[0] + if additional_ts is not None: + self.additional_ts = self.genTs(additional_ts=additional_ts)[2] if ts_value is not None: self.ts_value = ts_value @@ -1292,7 +1297,22 @@ class TDCom: for i in range(int(len(self.column_value_list)/2)): index_num = random.randint(0, len(self.column_value_list)-1) self.column_value_list[index_num] = None - self.column_value_list = [self.ts_value] + self.column_value_list + + if custom_col_index is not None: + if col_value_type == "Random": + pass + elif col_value_type == "Incremental": + self.column_value_list[custom_col_index] = self.custom_col_val + self.custom_col_val += 1 + elif col_value_type == "Part_equal": + self.column_value_list[custom_col_index] = random.choice(self.part_val_list) + + self.column_value_list = [self.ts_value] + [self.additional_ts] + self.column_value_list if additional_ts is not None else [self.ts_value] + self.column_value_list + if col_value_type == "Incremental" and custom_col_index==1: + self.column_value_list[custom_col_index] = self.custom_col_val if force_pk_val is None else force_pk_val + if col_value_type == "Part_equal" and custom_col_index==1: + self.column_value_list[custom_col_index] = random.randint(0, self.custom_col_val) if force_pk_val is None else force_pk_val + def screate_table(self, dbname=None, tbname="tb", use_name="table", column_elm_list=None, count=1, default_tbname_prefix="tb", default_tbname_index_start_num=1, @@ -1333,7 +1353,7 @@ class TDCom: default_tbname_index_start_num += 1 tdSql.execute(create_table_sql) - def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False): + def sinsert_rows(self, dbname=None, tbname=None, column_ele_list=None, ts_value=None, count=1, need_null=False, custom_col_index=None, col_value_type="random"): """insert rows Args: @@ -1353,7 +1373,7 @@ class TDCom: if tbname is not None: self.tbname = tbname - self.sgen_column_value_list(column_ele_list, need_null, ts_value) + self.sgen_column_value_list(column_ele_list, need_null, ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) # column_value_str = ", ".join(str(v) for v in self.column_value_list) column_value_str = "" for column_value in self.column_value_list: @@ -1370,7 +1390,7 @@ class TDCom: else: for num in range(count): ts_value = self.genTs()[0] - self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s') + self.sgen_column_value_list(column_ele_list, need_null, f'{ts_value}+{num}s', custom_col_index=custom_col_index, col_value_type=col_value_type) column_value_str = "" for column_value in self.column_value_list: if column_value is None: @@ -1777,7 +1797,7 @@ class TDCom: self.sdelete_rows(tbname=self.ctb_name, start_ts=self.time_cast(self.record_history_ts, "-")) self.sdelete_rows(tbname=self.tb_name, start_ts=self.time_cast(self.record_history_ts, "-")) - def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None): + def prepare_data(self, interval=None, watermark=None, session=None, state_window=None, state_window_max=127, interation=3, range_count=None, precision="ms", fill_history_value=0, ext_stb=None, custom_col_index=None, col_value_type="random"): """prepare stream data Args: @@ -1840,8 +1860,8 @@ class TDCom: if fill_history_value == 1: for i in range(self.range_count): ts_value = str(self.date_time)+f'-{self.default_interval*(i+1)}s' - self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) - self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) + self.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i == 1: self.record_history_ts = ts_value @@ -1862,6 +1882,18 @@ class TDCom: time.sleep(1) return tbname + def get_group_id_from_stb(self, stbname): + tdSql.query(f'select distinct group_id from {stbname}') + cnt = 0 + while len(tdSql.queryResult) == 0: + tdSql.query(f'select distinct group_id from {stbname}') + if cnt < self.default_interval: + cnt += 1 + time.sleep(1) + else: + return False + return tdSql.queryResult[0][0] + def update_json_file_replica(self, json_file_path, new_replica_value, output_file_path=None): """ Read a JSON file, update the 'replica' value, and write the result back to a file. diff --git a/tests/script/tsim/compress/compress_col.sim b/tests/script/tsim/compress/compress_col.sim index cc5b46fa06..ddf65b7f40 100644 --- a/tests/script/tsim/compress/compress_col.sim +++ b/tests/script/tsim/compress/compress_col.sim @@ -21,4 +21,15 @@ sql create table txx2 (ts timestamp compress 'disabled' level 'h', f int compres sql create table txx3 (ts timestamp compress 'disabled' level 'h', f int compress 'xz') sql create table txx4 (ts timestamp compress 'disabled' level 'h', f int compress 'zstd') + +sql alter table txx4 add column tt int compress 'xz' level 'h' +sql alter table txx4 drop column tt + +sql create stable sp(ts timestamp, c int) tags(t int) +sql alter table sp add column c1 int compress 'zstd' +sql alter table sp drop column c1 + +sql alter stable sp add column c1 int compress 'zstd' +sql alter stable sp drop column c1 + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 71aa763662..e4f29cc74e 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -49,7 +49,7 @@ print ============= create database with all options # | KEEP value [max(1d ~ 365000d), default: 1d, unit may be minut/hour/day] # | PRECISION ['ms' | 'us' | 'ns', default: ms] # | REPLICA value [1 | 3, default: 1] -# | WAL_LEVEL value [1 | 2, default: 1] +# | WAL_LEVEL value [0 | 1 | 2, default: 1] # | VGROUPS value [default: 2] # | SINGLE_STABLE [0 | 1, default: ] # @@ -404,7 +404,7 @@ endi sql drop database db sql_error create database db WAL_LEVEL 3 sql_error create database db WAL_LEVEL -1 -sql_error create database db WAL_LEVEL 0 +#sql_error create database db WAL_LEVEL 0 print ====> VGROUPS value [1~4096, default: 2] sql create database db VGROUPS 1 diff --git a/tests/script/tsim/insert/insert_select.sim b/tests/script/tsim/insert/insert_select.sim index 333964b1d6..888f349fbb 100644 --- a/tests/script/tsim/insert/insert_select.sim +++ b/tests/script/tsim/insert/insert_select.sim @@ -61,5 +61,23 @@ if $data02 != 1 then return -1 endi +sql insert into t2 (ts, b, a) select ts + 1, 11, 12 from t1; +sql select * from t2; +if $rows != 2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data12 != 11 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/create_db.sim b/tests/script/tsim/parser/create_db.sim index a5722f9819..5cb659586c 100644 --- a/tests/script/tsim/parser/create_db.sim +++ b/tests/script/tsim/parser/create_db.sim @@ -225,7 +225,7 @@ sql_error create database $db ctime 29 sql_error create database $db ctime 40961 # wal {0, 2} -sql_error create database testwal wal_level 0 +#sql_error create database testwal wal_level 0 sql select * from information_schema.ins_databases if $rows != 2 then return -1 diff --git a/tests/script/tsim/query/cache_last.sim b/tests/script/tsim/query/cache_last.sim index 50199117b0..b2d5ad8aa0 100644 --- a/tests/script/tsim/query/cache_last.sim +++ b/tests/script/tsim/query/cache_last.sim @@ -101,5 +101,248 @@ if $rows != 1 then return -1 endi +print step 2------------------------------- + +sql drop database if exists test; +sql create database test cachemodel 'both'; +sql use test; +sql create table stb (ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); + +sql create table t1 using stb tags(1,1,1); +sql create table t2 using stb tags(2,2,2); +sql insert into t1 values('2024-06-05 11:00:00',1,2,3); +sql insert into t1 values('2024-06-05 12:00:00',2,2,3); +sql insert into t2 values('2024-06-05 13:00:00',3,2,3); +sql insert into t2 values('2024-06-05 14:00:00',4,2,3); + +sql select last(ts) ts1,ts from stb; + +if $data00 != $data01 then + print $data00 + return -1 +endi + +sql select last(ts) ts1,ts from stb group by tbname; + +if $data00 != $data01 then + print $data00 + return -1 +endi + +sql select last(ts) ts1,tbname, ts from stb; + +if $data00 != $data02 then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +sql select last(ts) ts1,tbname, ts from stb group by tbname; + +if $data00 != $data02 then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +print step 3------------------------------- + +sql drop database if exists test1; +sql create database test1 cachemodel 'both'; +sql use test1; +sql create table stb (ts timestamp,a int primary key,b int,c int) tags(ta int,tb int,tc int); + +sql create table t1 using stb tags(1,1,1); +sql create table t2 using stb tags(2,2,2); +sql insert into t1 values('2024-06-05 11:00:00',1,2,3); +sql insert into t1 values('2024-06-05 12:00:00',2,2,3); +sql insert into t2 values('2024-06-05 13:00:00',3,2,3); +sql insert into t2 values('2024-06-05 14:00:00',4,2,3); + +sql select last(ts) ts1,ts from stb; + +if $data00 != $data01 then + print $data00 + return -1 +endi + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +sql select last(ts) ts1,ts from stb group by tbname; + +if $data00 != $data01 then + print $data00 + return -1 +endi + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +sql select last(ts) ts1,tbname, ts from stb; + +if $data00 != $data02 then + print $data00 + return -1 +endi + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +sql select last(ts) ts1,tbname, ts from stb group by tbname; + +if $data00 != $data02 then + print $data00 + return -1 +endi + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +print step 4------------------------------- + +sql select last(a) a,ts from stb; + +if $data00 != 4 then + print $data00 + return -1 +endi + +if $data01 != @24-06-05 14:00:00.000@ then + print $data01 + return -1 +endi + +sql select last(a) a,ts from stb group by tbname; + +if $data00 != 4 then + print $data00 + return -1 +endi + +if $data01 != @24-06-05 14:00:00.000@ then + print $data01 + return -1 +endi + +sql select last(a) a,tbname, ts from stb; + +if $data00 != 4 then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +if $data02 != @24-06-05 14:00:00.000@ then + print $data02 + return -1 +endi + +sql select last(a) a,tbname, ts from stb group by tbname; + +if $data00 != 4 then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +if $data02 != @24-06-05 14:00:00.000@ then + print $data02 + return -1 +endi + +print step 5------------------------------- + +sql select last(ts) ts1,a from stb; + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != 4 then + print $data01 + return -1 +endi + +sql select last(ts) ts1,a from stb group by tbname; + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != 4 then + print $data01 + return -1 +endi + +sql select last(ts) ts1,tbname, a from stb; + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +if $data02 != 4 then + print $data02 + return -1 +endi + +sql select last(ts) ts1,tbname, a from stb group by tbname; + +if $data00 != @24-06-05 14:00:00.000@ then + print $data00 + return -1 +endi + +if $data01 != t2 then + print $data01 + return -1 +endi + +if $data02 != 4 then + print $data02 + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/timeline.sim b/tests/script/tsim/query/timeline.sim new file mode 100644 index 0000000000..743a6f1a9f --- /dev/null +++ b/tests/script/tsim/query/timeline.sim @@ -0,0 +1,51 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql CREATE STABLE `demo` (`_ts` TIMESTAMP, `faev` DOUBLE) TAGS (`deviceid` VARCHAR(256)); +sql CREATE TABLE demo_1 USING demo (deviceid) TAGS ('1'); +sql CREATE TABLE demo_2 USING demo (deviceid) TAGS ('2'); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 1.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 01:00:00.001', 2.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-04 02:00:00.002', 3.0); +sql INSERT INTO demo_1 (_ts,faev) VALUES ('2023-12-05 03:00:00.003', 4.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-11-30 00:00:00.000', 5.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 01:00:00.001', 6.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-28 02:00:00.002', 7.0); +sql INSERT INTO demo_2 (_ts,faev) VALUES ('2023-12-29 03:00:00.003', 8.0); + +sql_error select diff(faev) from ((select ts, faev from demo union all select ts, faev from demo)); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts); +sql_error select diff(faev) from (select _ts, faev from demo union all select _ts, faev from demo order by faev, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev from demo union all select _ts + 1s, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo union all select _ts + 1s, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql_error select diff(faev) from (select _ts, faev from demo); +sql_error select diff(faev) from (select _ts, faev from demo order by faev, _ts); +sql select diff(faev) from (select _ts, faev from demo order by faev, _ts) partition by faev; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts) partition by deviceid; +sql_error select diff(faev) from (select _ts, faev, deviceid from demo order by deviceid, _ts) partition by faev; +sql select diff(faev) from (select _ts, faev, deviceid from demo order by faev, _ts, deviceid) partition by faev; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n))) ORDER BY ts, deviceid) PARTITION by deviceid; + + +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; +sql select deviceid, ts, diff(faev) as diff_faev FROM (SELECT deviceid, ts, faev FROM (SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0)UNION ALL(SELECT deviceid, _wstart AS ts, LAST(faev) AS faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-11-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid INTERVAL(1n)) ORDER BY ts, deviceid) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY deviceid, ts) PARTITION by deviceid; + +sql select deviceid, ts, diff(faev) as diff_faev FROM ((SELECT deviceid, ts, faev FROM (SELECT deviceid, _ts AS ts, faev, DIFF(ROUND(faev*1000)/1000) AS diff_faev FROM demo WHERE deviceid in ('201000008','K201000258') AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' PARTITION BY deviceid) WHERE diff_faev < 0)UNION ALL(SELECT deviceid, ts, faev FROM (SELECT deviceid, ts, faev, DIFF(ROUND(faev*1000)/1000) as diff_faev FROM (SELECT deviceid, _ts as ts , faev FROM demo WHERE deviceid in ('201000008','K201000258')AND _ts >= '2023-12-01 00:00:00' AND _ts < '2024-01-01 00:00:00' ORDER BY ts desc) PARTITION BY deviceid) WHERE diff_faev > 0) ORDER BY ts, deviceid) PARTITION by deviceid; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/all_insertmode_alltypes.json b/tests/system-test/0-others/all_insertmode_alltypes.json new file mode 100644 index 0000000000..dd5e373214 --- /dev/null +++ b/tests/system-test/0-others/all_insertmode_alltypes.json @@ -0,0 +1,248 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "connection_pool_size": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "prepared_rand": 10, + "chinese": "no", + "insert_interval": 0, + "num_of_records_per_req": 10, + "databases": [{ + "dbinfo": { + "name": "db_all_insert_mode", + "drop": "yes" + }, + "super_tables": [{ + "name": "sml_json", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb1_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml", + "line_protocol": "json", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "BOOL"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "sml_line", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb2_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml", + "line_protocol": "line", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "TINYINT"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "sml_telnet", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb3_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml", + "line_protocol": "telnet", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "SMALLINT"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "rest", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb4_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "rest", + "line_protocol": "json", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "stmt", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb5_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "stmt", + "line_protocol": "json", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "BIGINT"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "sml_rest_json", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb6_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml-rest", + "line_protocol": "json", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "FLOAT"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "sml_rest_line", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb7_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml-rest", + "line_protocol": "line", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "DOUBLE"}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "sml_rest_telnet", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb8_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "sml-rest", + "line_protocol": "telnet", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "BINARY", "len": 8}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + },{ + "name": "taosc", + "child_table_exists":"no", + "childtable_count": 8, + "childtable_prefix": "stb9_", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "line_protocol": "json", + "childtable_limit": 0, + "childtable_offset": 0, + "insert_rows": 2, + "insert_interval": 0, + "interlace_rows": 0, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "now", + "sample_file": "", + "use_sample_ts": "no", + "tags_file": "", + "columns": [{"type": "NCHAR", "len": 8}], + "tags": [{"type": "INT"}, {"type": "BIGINT"}, {"type": "FLOAT"}, {"type": "DOUBLE"}, {"type": "SMALLINT"}, {"type": "TINYINT"}, {"type": "BOOL"}, {"type": "NCHAR","len": 17, "count":1}, {"type": "BINARY", "len": 19, "count":1}] + }] + }] +} diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index 71adf6eaac..9ba3bd0d2f 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -43,6 +43,7 @@ class TDTestCase: delete from deldata.ct1; insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ); flush database deldata;''' + def checkProcessPid(self,processName): i=0 while i<60: @@ -110,7 +111,8 @@ class TDTestCase: else: print(f"{packageName} has been exists") os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) - tdDnodes.stop(1) + + os.system(f"pkill -9 taosd" ) print(f"start taosd: rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & ") os.system(f"rm -rf {dataPath}/* && nohup /usr/bin/taosd -c {cPath} & " ) os.system(f"killall taosadapter" ) @@ -165,32 +167,31 @@ class TDTestCase: cPath = self.getCfgPath() dbname = "test" stb = f"{dbname}.meters" + # package_type = "enterprise" package_type = "community" self.installTaosd(bPath,cPath,package_type) # os.system(f"echo 'debugFlag 143' >> {cPath}/taos.cfg ") tableNumbers=100 - recordNumbers1=100 + recordNumbers1=1000 recordNumbers2=1000 - # tdsqlF=tdCom.newTdSql() - # print(tdsqlF) - # tdsqlF.query(f"SELECT SERVER_VERSION();") - # print(tdsqlF.query(f"SELECT SERVER_VERSION();")) - # oldServerVersion=tdsqlF.queryResult[0][0] - # tdLog.info(f"Base server version is {oldServerVersion}") - # tdsqlF.query(f"SELECT CLIENT_VERSION();") - # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison - # oldClientVersion=tdsqlF.queryResult[0][0] - # tdLog.info(f"Base client version is {oldClientVersion}") - # baseVersion = "3.0.1.8" - tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}") - tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ") - os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -y ") + tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -O 5 -y ") + os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -v 1 -O 5 -y ") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database test keep 365000 '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database test cachemodel \"both\" '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from test.meters '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s \"insert into test.d1 values (now+1s, 11, 190, 0.21), (now+2s, 11, 190, 0.21), (now+3s, 11, 190, 0.21), ('2015-07-14 08:39:59.001', 11, 190, 0.21), ('2032-08-14 08:39:59.001 ', 11, 190, 0.21) test.d3 values (now+6s, 11, 190, 0.21), (now+7s, 11, 190, 0.21), (now+8s, 11, 190, 0.21), ('2033-07-14 08:39:59.000', 119, 191, 0.25) test.d3 (ts) values ('2033-07-14 08:39:58.000');\"") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from test.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '") + + os.system("LD_LIBRARY_PATH=/usr/lib taos -s \"insert into test.d1 values (now+11s, 11, 190, 0.21), (now+12s, 11, 190, 0.21), (now+13s, 11, 190, 0.21), (now+14s, 11, 190, 0.21), (now+15s, 11, 190, 0.21) test.d3 values (now+16s, 11, 190, 0.21), (now+17s, 11, 190, 0.21), (now+18s, 11, 190, 0.21), (now+19s, 119, 191, 0.25) test.d3 (ts) values (now+20s);\"") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/com_alltypedata.json -y") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database curdb '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'alter database curdb cachemodel \"both\" '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select count(*) from curdb.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select last(*) from curdb.meters '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select sum(fc) from curdb.meters '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select avg(ic) from curdb.meters '") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select min(ui) from curdb.meters '") @@ -211,7 +212,8 @@ class TDTestCase: os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {stable_topic} as stable test.meters where tbname like \\"d3\\";" ') select_topic = "select_test_meters_topic" - os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {select_topic} as select current,voltage,phase from test.meters where voltage >= 170;" ') + topic_select_sql = "select current,voltage,phase from test.meters where voltage >= 10;" + os.system(f'LD_LIBRARY_PATH=/usr/lib taos -s "create topic if not exists {select_topic} as {topic_select_sql}" ') os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;show topics;" ') os.system(f" /usr/bin/taosadapter --version " ) @@ -243,37 +245,12 @@ class TDTestCase: break consumer.close() - # consumer_dict2 = { - # "group.id": "g2", - # "td.connect.websocket.scheme": "ws", - # "td.connect.user": "root", - # "td.connect.pass": "taosdata", - # "auto.offset.reset": "earliest", - # "enable.auto.commit": "false", - # } - # consumer = taosws.Consumer(consumer_dict2) - # try: - # consumer.subscribe([db_topic,stable_topic]) - # except TmqError: - # tdLog.exit(f"subscribe error") - # first_consumer_rows = 0 - # while True: - # message = consumer.poll(timeout=1.0) - # if message: - # for block in message: - # first_consumer_rows += block.nrows() - # else: - # tdLog.notice("message is null and break") - # break - # consumer.commit(message) - # tdLog.debug(f"topic:{select_topic} ,first consumer rows is {first_consumer_rows} in old version") - # break - - - + tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/all_insertmode_alltypes.json -y") + + # os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database db4096 '") os.system("LD_LIBRARY_PATH=/usr/lib taos -f 0-others/TS-3131.tsql") # add deleted data @@ -285,9 +262,10 @@ class TDTestCase: if os.system(cmd) == 0: raise Exception("failed to execute system command. cmd: %s" % cmd) - os.system("pkill taosd") # make sure all the data are saved in disk. + os.system("pkill -9 taosd") # make sure all the data are saved in disk. + os.system("pkill -9 taos") self.checkProcessPid("taosd") - os.system("pkill taosadapter") # make sure all the data are saved in disk. + os.system("pkill -9 taosadapter") # make sure all the data are saved in disk. self.checkProcessPid("taosadapter") tdLog.printNoPrefix("==========step2:update new version ") @@ -309,13 +287,33 @@ class TDTestCase: tdLog.info(f"New client version is {nowClientVersion}") tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql.query(f"select last(*) from curdb.meters") + tdLog.info(tdsql.queryResult) + tdsql.query(f"select * from db_all_insert_mode.sml_json") + tdsql.checkRows(16) + + tdsql.query(f"select * from db_all_insert_mode.sml_line") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.sml_telnet") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.rest") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.stmt") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.sml_rest_json") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.sml_rest_line") + tdsql.checkRows(16) + tdsql.query(f"select * from db_all_insert_mode.sml_rest_telnet") + tdsql.checkRows(16) + tdsql.query(f"select count(*) from {stb}") - tdsql.checkData(0,0,tableNumbers*recordNumbers1) + tdsql.checkData(0,0,tableNumbers*recordNumbers1+20) tdsql.query("show streams;") tdsql.checkRows(2) - + # checkout db4096 tdsql.query("select count(*) from db4096.stb0") tdsql.checkData(0,0,50000) @@ -381,9 +379,78 @@ class TDTestCase: tdLog.exit("The unordered list is not the same as the ordered list.") + # check database test and last + # first check + + tdsql.query(f"select last(*) from test.meters group by tbname") + tdLog.info(tdsql.queryResult) + # tdsql.checkRows(tableNumbers) + + tdsql.query(f"select last_row(*) from test.meters group by tbname") + tdLog.info(tdsql.queryResult) + # tdsql.checkRows(tableNumbers) + + tdsql.query(f"select last_row(*) from test.meters partition by tbname") + tdLog.info(tdsql.queryResult) + # tdsql.checkRows(tableNumbers) + + tdsql.query(f"select last(*) from test.meters") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:39:59.000") + tdsql.checkData(0,1,119) + tdsql.checkData(0,2,191) + tdsql.checkData(0,3,0.25) + + tdsql.query(f"select last_row(*) from test.meters") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:39:59.000") + tdsql.checkData(0,1,119) + tdsql.checkData(0,2,191) + tdsql.checkData(0,3,0.25) + + tdsql.query(f"select last(*) from test.d1") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2032-08-14 08:39:59.001") + tdsql.checkData(0,1,11) + tdsql.checkData(0,2,190) + tdsql.checkData(0,3,0.21) + + # update data and check + tdsql.execute("insert into test.d2 values ('2033-07-14 08:39:59.002', 139, 182, 1.10) (now+2s, 12, 191, 0.22) test.d2 (ts) values ('2033-07-14 08:39:59.003');") + tdsql.execute("insert into test.d2 values (now+5s, 4.3, 104, 0.4);") + + tdsql.query(f"select last(*) from test.meters") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:39:59.003") + tdsql.checkData(0,1,139) + tdsql.checkData(0,2,182) + tdsql.checkData(0,3,1.10) + + # repeately insert data and check + tdsql.execute("insert into test.d1 values (now+1s, 11, 190, 0.21) (now+2s, 12, 191, 0.22) ('2033-07-14 08:40:01.001', 16, 180, 0.53);") + + tdsql.query(f"select last(*) from test.d1") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:40:01.001") + tdsql.checkData(0,1,16) + tdsql.checkData(0,2,180) + tdsql.checkData(0,3,0.53) + + tdsql.query(f"select last(*) from test.meters") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:40:01.001") + tdsql.checkData(0,1,16) + tdsql.checkData(0,2,180) + tdsql.checkData(0,3,0.53) + + tdsql.query(f"select last_row(*) from test.meters") + tdLog.info(tdsql.queryResult) + tdsql.checkData(0,0,"2033-07-14 08:40:01.001") + tdsql.checkData(0,1,16) + tdsql.checkData(0,2,180) + tdsql.checkData(0,3,0.53) + # check tmq - tdsql.execute("insert into test.d80 values (now+1s, 11, 190, 0.21);") - tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") conn = taos.connect() consumer = Consumer( @@ -408,7 +475,7 @@ class TDTestCase: print("consumer has completed and break") break consumer.close() - tdsql.query("select current,voltage,phase from test.meters where voltage >= 170;") + tdsql.query(f"{topic_select_sql}") all_rows = tdsql.queryRows if consumer_rows < all_rows - first_consumer_rows : tdLog.exit(f"consumer rows is {consumer_rows}, less than {all_rows - first_consumer_rows}") diff --git a/tests/system-test/0-others/empty_identifier.py b/tests/system-test/0-others/empty_identifier.py new file mode 100644 index 0000000000..a576781d86 --- /dev/null +++ b/tests/system-test/0-others/empty_identifier.py @@ -0,0 +1,180 @@ +import taos +import sys +import time +import socket +import os +import threading +import math +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +COMPARE_DATA = 0 +COMPARE_LEN = 1 + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, + {'type': 'BIGINT', 'count':1}, + {'type': 'FLOAT', 'count':1}, + {'type': 'DOUBLE', 'count':1}, + {'type': 'smallint', 'count':1}, + {'type': 'tinyint', 'count':1}, + {'type': 'bool', 'count':1}, + {'type': 'binary', 'len':10, 'count':1}, + {'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def run(self): + self.prepareTestEnv() + self.test_empty_identifier() + + def execute_sql_and_expect_err(self, sql: str, err: int): + tdSql.error(sql, err) + + def test_empty_identifier(self): + ## invalid identifier + sqls = [ + 'show create table ``', + 'show create table test.``', + 'create table `` (ts timestamp, c1 int)', + 'drop table ``', + 'alter table `` add column c2 int', + 'select * from ``', + 'alter table meters add column `` int', + 'alter table meters drop column ``', + 'alter table t0 set tag `` = ""', + 'alter stable meters add tag `` int', + 'alter stable meters rename tag cc ``', + 'alter stable meters drop tag ``', + 'insert into `` select * from t0', + 'insert into t100 using `` tags('', '') values(1,1,1)', + 'create view `` as select count(*) from meters interval(10s)', + 'create view ``.view1 as select count(*) from meters' + 'create tsma `` on meters function(count(c1)) interval(1m)', + 'create tsma tsma1 on `` function(count(c1)) interval(1m)', + 'create stream `` into st1 as select count(*) from meters interval(10s)', + 'create stream stream1 into `` as select count(*) from meters interval(10s)', + 'create stream stream1 into st1 as select count(*) from `` interval(10s)', + 'create stream stream1 trigger max_delay 100s into st1 as select count(*) from `` interval(10s)', + 'drop view ``', + 'drop tsma ``', + 'drop view ``.st1', + 'create topic `` as select count(*) from meters interval(10s)', + 'drop topic ``', + 'insert into `` values(1,1,1)', + ] + + for sql in sqls: + self.execute_sql_and_expect_err(sql, -2147473897) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/fsync.py b/tests/system-test/0-others/fsync.py index 04a9f5c941..a1f51d1d1e 100644 --- a/tests/system-test/0-others/fsync.py +++ b/tests/system-test/0-others/fsync.py @@ -148,7 +148,7 @@ class TDTestCase: @property def fsync_create_err(self): return [ - "create database db1 wal_level 0", + #"create database db1 wal_level 0", "create database db1 wal_level 3", "create database db1 wal_level null", "create database db1 wal_level true", @@ -162,7 +162,7 @@ class TDTestCase: @property def fsync_alter_err(self): return [ - "alter database db1 wal_level 0", + #"alter database db1 wal_level 0", "alter database db1 wal_level 3", "alter database db1 wal_level null", "alter database db1 wal_level true", diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 944b2fbb1e..0d9f43424d 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,7 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(254, 255)) + tdSql.checkEqual(True, len(tdSql.queryResult) in range(260, 261)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(54, len(tdSql.queryResult)) @@ -269,6 +269,10 @@ class TDTestCase: def ins_grants_check(self): grant_name_dict = { + 'service':'Service Time', + 'timeseries':'Timeseries', + 'dnodes':'Dnodes', + 'cpu_cores':'CPU Cores', 'stream':'Stream', 'subscription':'Subscription', 'view':'View', @@ -293,6 +297,7 @@ class TDTestCase: 'mysql':'MySQL', 'postgres':'PostgreSQL', 'oracle':'Oracle', + 'mssql':'SqlServer' } tdSql.execute('drop database if exists db2') @@ -304,7 +309,7 @@ class TDTestCase: if result[i][0] in grant_name_dict: tdSql.checkEqual(result[i][1], grant_name_dict[result[i][0]]) index += 1 - tdSql.checkEqual(index, 24) + tdSql.checkEqual(index, len(grant_name_dict)) tdSql.query(f'select * from information_schema.ins_grants_logs') result = tdSql.queryResult tdSql.checkEqual(True, len(result) >= 0) @@ -336,7 +341,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[1]) index += 1 tdSql.checkEqual(True, index > 0) - + tdSql.query(f'show encryptions') result = tdSql.queryResult index = 0 @@ -344,7 +349,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[1]) index += 1 tdSql.checkEqual(True, index > 0) - + # loaded/sm4 tdSql.execute('drop database if exists db2') tdSql.execute('create encrypt_key \'12345678\'') @@ -357,7 +362,7 @@ class TDTestCase: tdSql.checkEqual(True, result[i][1] in key_status_list[3]) index += 1 tdSql.checkEqual(True, index > 0) - + tdSql.query(f'show encryptions') result = tdSql.queryResult index = 0 diff --git a/tests/system-test/0-others/show.py b/tests/system-test/0-others/show.py index a5db5d193a..64696c5e6d 100644 --- a/tests/system-test/0-others/show.py +++ b/tests/system-test/0-others/show.py @@ -186,17 +186,17 @@ class TDTestCase: tdSql.query('show dnode 1 variables') for i in tdSql.queryResult: if i[1].lower() == "gitinfo": - taosd_gitinfo_sql = f"gitinfo: {i[2]}" + taosd_gitinfo_sql = f"git: {i[2]}" taos_gitinfo_sql = '' tdSql.query('show local variables') for i in tdSql.queryResult: if i[0].lower() == "gitinfo": - taos_gitinfo_sql = f"gitinfo: {i[1]}" + taos_gitinfo_sql = f"git: {i[1]}" taos_info = os.popen('taos -V').read() - taos_gitinfo = re.findall("^gitinfo.*",taos_info,re.M) + taos_gitinfo = re.findall("^git: .*",taos_info,re.M) tdSql.checkEqual(taos_gitinfo_sql,taos_gitinfo[0]) taosd_info = os.popen('taosd -V').read() - taosd_gitinfo = re.findall("^gitinfo.*",taosd_info,re.M) + taosd_gitinfo = re.findall("^git: .*",taosd_info,re.M) tdSql.checkEqual(taosd_gitinfo_sql,taosd_gitinfo[0]) def show_base(self): diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 1227378799..3b6f6c6fab 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -5,6 +5,7 @@ import time import socket import os import platform +import re if platform.system().lower() == 'windows': import wexpect as taosExpect else: @@ -370,10 +371,11 @@ class TDTestCase: if retCode != "TAOS_OK": tdLog.exit("taos -V fail") - version = 'version: ' + version - retVal = retVal.replace("\n", "") - retVal = retVal.replace("\r", "") - if retVal.startswith(version) == False: + version = 'taos version: ' + version + # retVal = retVal.replace("\n", "") + # retVal = retVal.replace("\r", "") + taosVersion = re.findall((f'^%s'%(version)), retVal,re.M) + if len(taosVersion) == 0: print ("return version: [%s]"%retVal) print ("dict version: [%s]"%version) tdLog.exit("taos -V version not match") diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index 21817ef20d..493e1491b8 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -20,6 +20,7 @@ from util.common import * from util.sqlset import * class TDTestCase: + updatecfgDict = {'stdebugflag':143} def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) diff --git a/tests/system-test/1-insert/insert_double.py b/tests/system-test/1-insert/insert_double.py index b7af7237db..d6dada32cd 100644 --- a/tests/system-test/1-insert/insert_double.py +++ b/tests/system-test/1-insert/insert_double.py @@ -26,30 +26,30 @@ class TDTestCase: tdSql.execute(f"drop table if exists {table_name}") tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)") - tdSql.execute(f"insert into {table_name} values(now, -16, +6)") - tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)") - tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)") - tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)") - tdSql.execute(f"insert into {table_name} values(now, -12., +3.)") - tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)") - tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)") - tdSql.execute(f"insert into {table_name} values(now, -2e1, +2e2)") - tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)") - tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)") - tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)") + tdSql.execute(f"insert into {table_name} values(1717122943000, -16, +6)") + tdSql.execute(f"insert into {table_name} values(1717122944000, 80.99, +0042)") + tdSql.execute(f"insert into {table_name} values(1717122945000, -0042, +80.99)") + tdSql.execute(f"insert into {table_name} values(1717122946000, 52.34354, 18.6)") + tdSql.execute(f"insert into {table_name} values(1717122947000, -12., +3.)") + tdSql.execute(f"insert into {table_name} values(1717122948000, -0.12, +3.0)") + tdSql.execute(f"insert into {table_name} values(1717122949000, -2.3e1, +2.324e2)") + tdSql.execute(f"insert into {table_name} values(1717122950000, -2e1, +2e2)") + tdSql.execute(f"insert into {table_name} values(1717122951000, -2.e1, +2.e2)") + tdSql.execute(f"insert into {table_name} values(1717122952000, -0x40, +0b10000)") + tdSql.execute(f"insert into {table_name} values(1717122953000, -0b10000, +0x40)") # str support - tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')") - tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')") - tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')") - tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')") - tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')") - tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')") - tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')") - tdSql.execute(f"insert into {table_name} values(now, '-2e1', '+2e2')") - tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')") - tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')") - tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')") + tdSql.execute(f"insert into {table_name} values(1717122954000, '-16', '+6')") + tdSql.execute(f"insert into {table_name} values(1717122955000, ' -80.99', ' +0042')") + tdSql.execute(f"insert into {table_name} values(1717122956000, ' -0042', ' +80.99')") + tdSql.execute(f"insert into {table_name} values(1717122957000, '52.34354', '18.6')") + tdSql.execute(f"insert into {table_name} values(1717122958000, '-12.', '+5.')") + tdSql.execute(f"insert into {table_name} values(1717122959000, '-.12', '+.5')") + tdSql.execute(f"insert into {table_name} values(1717122960000, '-2.e1', '+2.e2')") + tdSql.execute(f"insert into {table_name} values(1717122961000, '-2e1', '+2e2')") + tdSql.execute(f"insert into {table_name} values(1717122962000, '-2.3e1', '+2.324e2')") + tdSql.execute(f"insert into {table_name} values(1717122963000, '-0x40', '+0b10010')") + tdSql.execute(f"insert into {table_name} values(1717122964000, '-0b10010', '+0x40')") tdSql.query(f"select * from {table_name}") tdSql.checkRows(22) @@ -64,22 +64,22 @@ class TDTestCase: min_u = 0 print("val:", baseval, negval, posval, max_i) - tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})") - tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})") - tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})") - tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})") + tdSql.execute(f"insert into {table_name} values(1717122965000, {negval}, {posval})") + tdSql.execute(f"insert into {table_name} values(1717122966000, -{baseval}, {baseval})") + tdSql.execute(f"insert into {table_name} values(1717122967000, {max_i}, {max_u})") + tdSql.execute(f"insert into {table_name} values(1717122968000, {min_i}, {min_u})") tdSql.query(f"select * from {table_name}") tdSql.checkRows(26) # error case - tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})") - tdSql.error(f"insert into {table_name} values(now, 0, -1)") - tdSql.error(f"insert into {table_name} values(now, 0, -2.0)") - tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')") - tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)") - tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)") - tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)") + tdSql.error(f"insert into {table_name} values(1717122969000, 0, {max_u+1})") + tdSql.error(f"insert into {table_name} values(1717122970000, 0, -1)") + tdSql.error(f"insert into {table_name} values(1717122971000, 0, -2.0)") + tdSql.error(f"insert into {table_name} values(1717122972000, 0, '-2.0')") + tdSql.error(f"insert into {table_name} values(1717122973000, {max_i+1}, 0)") + tdSql.error(f"insert into {table_name} values(1717122974000, {min_i-1}, 0)") + tdSql.error(f"insert into {table_name} values(1717122975000, '{min_i-1}', 0)") def test_tags(self, stable_name, dtype, bits): tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})") @@ -93,20 +93,20 @@ class TDTestCase: max_u = 2*bigval - 1 min_u = 0 - tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})") - tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})") - tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})") - tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})") + tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(1717122976000, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(1717122977000, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(1717122978000, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(1717122979000, {min_i}, {min_u})") - tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')") - tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')") - tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')") - tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')") + tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(1717122980000, '{negval}', '{posval}')") + tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(1717122981000, '-{baseval}' , '{baseval}')") + tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(1717122982000, '{max_i}', '{max_u}')") + tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(1717122983000, '{min_i}', '{min_u}')") - tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})") - tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})") - tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})") - tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})") + tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(1717122984000, {negval}, {posval})") + tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(1717122985000, -{baseval} , {baseval})") + tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(1717122986000, {max_i}, {max_u})") + tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(1717122987000, {min_i}, {min_u})") tdSql.query(f"select * from {stable_name}") tdSql.checkRows(12) diff --git a/tests/system-test/2-query/avg.py b/tests/system-test/2-query/avg.py index dd8b38f9ea..34e4a19978 100644 --- a/tests/system-test/2-query/avg.py +++ b/tests/system-test/2-query/avg.py @@ -436,13 +436,13 @@ class TDTestCase: tdSql.execute(f"use db") tdSql.execute(f"create table db.st(ts timestamp, ibv bigint, ubv bigint unsigned) tags(area int)") # insert t1 data - tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,9223372036854775801,18446744073709551611)") - tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,8223372036854775801,17446744073709551611)") - tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now,7223372036854775801,16446744073709551611)") + tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 1s,9223372036854775801,18446744073709551611)") + tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 2s,8223372036854775801,17446744073709551611)") + tdSql.execute(f"insert into db.t1 using db.st tags(1) values(now + 3s,7223372036854775801,16446744073709551611)") # insert t2 data - tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,9223372036854775801,18446744073709551611)") - tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,8223372036854775801,17446744073709551611)") - tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now,7223372036854775801,16446744073709551611)") + tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 1s,9223372036854775801,18446744073709551611)") + tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 2s,8223372036854775801,17446744073709551611)") + tdSql.execute(f"insert into db.t2 using db.st tags(2) values(now + 3s,7223372036854775801,16446744073709551611)") # check single table answer tdSql.query(f"select avg(ibv), avg(ubv) from db.t1") diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index 155cb39b62..79196b7fe3 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -257,10 +257,10 @@ class TDTestCase: os.system(f'taos -f {sql_file}') tdSql.query('select count(c_1) from d2.t2 where c_1 < 10', queryTimes=1) tdSql.checkData(0, 0, 0) - tdSql.query('select count(c_1), min(c_1),tbname from d2.can partition by tbname', queryTimes=1) + tdSql.query('select count(c_1), min(c_1),tbname from d2.can partition by tbname order by 3', queryTimes=1) tdSql.checkData(0, 0, 0) tdSql.checkData(0, 1, None) - tdSql.checkData(0, 2, 't3') + tdSql.checkData(0, 2, 't1') tdSql.checkData(1, 0, 15) tdSql.checkData(1, 1, 1471617148940980000) @@ -268,7 +268,7 @@ class TDTestCase: tdSql.checkData(2, 0, 0) tdSql.checkData(2, 1, None) - tdSql.checkData(2, 2, 't1') + tdSql.checkData(2, 2, 't3') def run(self): self.test_count_with_sma_data() diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index 4b236c1bce..c63b8af9df 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -237,6 +237,9 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and t2=0;") tdSql.checkRows(1) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and t2=0 having count(*) > 10;") + tdSql.checkRows(0) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _rowts>0;") tdSql.checkRows(nonempty_tb_num) diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 81c5b66185..1cb95b59c5 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -15,6 +15,30 @@ class TDTestCase: #tdSql.init(conn.cursor()) tdSql.init(conn.cursor(), logSql) # output sql.txt file + def interp_on_empty_table(self): + dbname = "db" + tbname = "t" + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute(f'''create table if not exists {dbname}.{tbname} (ts timestamp, k int)''') + + tdLog.printNoPrefix("==========step2:interp query on empty table") + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(prev)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(next)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(linear)") + tdSql.checkRows(0) + + tdSql.query(f"select _irowts, interp(k),k from {dbname}.{tbname} partition by k range(now()-1h, now()) every(1m) fill(value, 2)") + tdSql.checkRows(0) + def run(self): dbname = "db" tbname = "tb" @@ -5658,6 +5682,7 @@ class TDTestCase: tdSql.checkData(0, 0, '2023-08-06 23:59:00') tdSql.checkData(0, 1, None) + self.interp_on_empty_table() def stop(self): diff --git a/tests/system-test/2-query/td-28068.py b/tests/system-test/2-query/td-28068.py index 0dfaf8e126..0a7e75fef2 100644 --- a/tests/system-test/2-query/td-28068.py +++ b/tests/system-test/2-query/td-28068.py @@ -10,14 +10,14 @@ class TDTestCase: tdSql.execute("create database td_28068;") tdSql.execute("create database if not exists td_28068;") tdSql.execute("create stable td_28068.st (ts timestamp, test_case nchar(10), time_cost float, num float) tags (branch nchar(10), scenario nchar(10));") - tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (now(), 'query1', 1,2);") - tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (now(), 'query1', 2,3);") - tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (now(), 'query1', 10,1);") - tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (now(), 'query1', 11,5);") - tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (now(), 'query1', 20,4);") - tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (now(), 'query1', 30,1);") - tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (now(), 'query1', 8,8);") - tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (now(), 'query1', 9,10);") + tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (1717122943000, 'query1', 1,2);") + tdSql.execute("insert into td_28068.ct1 using td_28068.st (branch, scenario) tags ('3.0', 'scenario1') values (1717122944000, 'query1', 2,3);") + tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (1717122945000, 'query1', 10,1);") + tdSql.execute("insert into td_28068.ct2 using td_28068.st (branch, scenario) tags ('3.0', 'scenario2') values (1717122946000, 'query1', 11,5);") + tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (1717122947000, 'query1', 20,4);") + tdSql.execute("insert into td_28068.ct3 using td_28068.st (branch, scenario) tags ('3.1', 'scenario1') values (1717122948000, 'query1', 30,1);") + tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (1717122949000, 'query1', 8,8);") + tdSql.execute("insert into td_28068.ct4 using td_28068.st (branch, scenario) tags ('3.1', 'scenario2') values (1717122950000, 'query1', 9,10);") def run(self): tdSql.query('select last(ts) as ts, last(branch) as branch, last(scenario) as scenario, last(test_case) as test_case from td_28068.st group by branch, scenario order by last(branch);') diff --git a/tests/system-test/2-query/union1.py b/tests/system-test/2-query/union1.py index 8db5ce01f3..853dfe9582 100644 --- a/tests/system-test/2-query/union1.py +++ b/tests/system-test/2-query/union1.py @@ -240,9 +240,22 @@ class TDTestCase: tdSql.error( " '' union all select c1 from ct1 " ) # tdSql.error( "select c1 from ct1 union select c1 from ct2 union select c1 from ct4 ") + def test_select_from_union_all(self): + tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all select ts, c8, c1 from stb1 limit 15)') + tdSql.checkRows(15) + tdSql.query('select c8, ts from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c1 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c1, c8 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + tdSql.query('select ts, c8, c1, 123 from ((select ts, c8,c1 from stb1 order by c1) union all (select ts, c8, c1 from stb1 order by c8 limit 10) limit 15)') + tdSql.checkRows(15) + def all_test(self): self.__test_error() self.union_check() + self.test_select_from_union_all() def __create_tb(self): diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 94e9babf3c..39b9c6d34a 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -278,6 +278,19 @@ class TDTestCase: self.checkDropData(False) return + + def checkSnapshot1VgroupBtmeta(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -bt'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot") + self.checkData() + self.checkDropData(False) + + return def checkSnapshot1VgroupTable(self): buildPath = tdCom.getBuildPath() @@ -290,6 +303,18 @@ class TDTestCase: self.checkDataTable() return + + def checkSnapshot1VgroupTableBtmeta(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s -sv 1 -dv 1 -s -t -bt'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkJson(cfgPath, "tmq_taosx_tmp_snapshot") + self.checkDataTable() + + return def checkSnapshotMultiVgroups(self): buildPath = tdCom.getBuildPath() @@ -301,6 +326,17 @@ class TDTestCase: self.checkDropData(False) return + + def checkSnapshotMultiVgroupsBtmeta(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -bt'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkData() + self.checkDropData(False) + + return def checkSnapshotMultiVgroupsWithDropTable(self): buildPath = tdCom.getBuildPath() @@ -311,6 +347,16 @@ class TDTestCase: self.checkDropData(True) return + + def checkSnapshotMultiVgroupsWithDropTableBtmeta(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -sv 2 -dv 4 -s -d -bt'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkDropData(True) + + return def consumeTest(self): tdSql.execute(f'create database if not exists d1 vgroups 1') @@ -472,6 +518,11 @@ class TDTestCase: self.checkSnapshotMultiVgroupsWithDropTable() + self.checkSnapshot1VgroupBtmeta() + self.checkSnapshot1VgroupTableBtmeta() + self.checkSnapshotMultiVgroupsBtmeta() + self.checkSnapshotMultiVgroupsWithDropTableBtmeta() + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/8-stream/at_once_interval.py b/tests/system-test/8-stream/at_once_interval.py index 763b88bc69..eb581e84c4 100644 --- a/tests/system-test/8-stream/at_once_interval.py +++ b/tests/system-test/8-stream/at_once_interval.py @@ -15,9 +15,12 @@ class TDTestCase: def at_once_interval(self, interval, partition="tbname", delete=False, fill_value=None, fill_history_value=None, case_when=None): tdLog.info(f"*** testing stream at_once+interval: interval: {interval}, partition: {partition}, fill_history: {fill_history_value}, fill: {fill_value}, delete: {delete}, case_when: {case_when} ***") + col_value_type = "Incremental" if partition=="c1" else "random" + custom_col_index = 1 if partition=="c1" else None + self.tdCom.custom_col_val = 0 self.delete = delete self.tdCom.case_name = sys._getframe().f_code.co_name - self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value) + self.tdCom.prepare_data(interval=interval, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type) self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "") self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "") self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "") @@ -76,15 +79,15 @@ class TDTestCase: for i in range(self.tdCom.range_count): ts_value = str(self.tdCom.date_time+self.tdCom.dataDict["interval"])+f'+{i*10}s' ts_cast_delete_value = self.tdCom.time_cast(ts_value) - self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.ctb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: self.tdCom.sdelete_rows(tbname=self.tdCom.ctb_name, start_ts=ts_cast_delete_value) self.tdCom.date_time += 1 - self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value) + self.tdCom.sinsert_rows(tbname=self.tdCom.tb_name, ts_value=ts_value, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: self.tdCom.sdelete_rows(tbname=self.tdCom.tb_name, start_ts=ts_cast_delete_value) self.tdCom.date_time += 1 @@ -102,6 +105,7 @@ class TDTestCase: if self.tdCom.subtable: for tname in [self.stb_name, self.ctb_name]: + group_id = self.tdCom.get_group_id_from_stb(f'{tname}_output') tdSql.query(f'select * from {self.ctb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -116,11 +120,11 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{tname}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{tname}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) - + group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output') tdSql.query(f'select * from {self.tb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -135,7 +139,7 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{abs_c1_value}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 diff --git a/tests/system-test/8-stream/at_once_session.py b/tests/system-test/8-stream/at_once_session.py index 6f25e5ad97..cdded9388c 100644 --- a/tests/system-test/8-stream/at_once_session.py +++ b/tests/system-test/8-stream/at_once_session.py @@ -15,9 +15,12 @@ class TDTestCase: def at_once_session(self, session, ignore_expired=None, ignore_update=None, partition="tbname", delete=False, fill_history_value=None, case_when=None, subtable=True): tdLog.info(f"*** testing stream at_once+interval: session: {session}, ignore_expired: {ignore_expired}, ignore_update: {ignore_update}, partition: {partition}, delete: {delete}, fill_history: {fill_history_value}, case_when: {case_when}, subtable: {subtable} ***") + col_value_type = "Incremental" if partition=="c1" else "random" + custom_col_index = 1 if partition=="c1" else None + self.tdCom.custom_col_val = 0 self.delete = delete self.tdCom.case_name = sys._getframe().f_code.co_name - self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value) + self.tdCom.prepare_data(session=session, fill_history_value=fill_history_value, custom_col_index=custom_col_index, col_value_type=col_value_type) self.stb_name = self.tdCom.stb_name.replace(f"{self.tdCom.dbname}.", "") self.ctb_name = self.tdCom.ctb_name.replace(f"{self.tdCom.dbname}.", "") self.tb_name = self.tdCom.tb_name.replace(f"{self.tdCom.dbname}.", "") @@ -79,11 +82,11 @@ class TDTestCase: if i == 0: record_window_close_ts = window_close_ts for ts_value in [self.tdCom.date_time, window_close_ts]: - self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True) - self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True) + self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.tdCom.update and i%2 == 0: - self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True) - self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True) + self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) + self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value, need_null=True, custom_col_index=custom_col_index, col_value_type=col_value_type) if self.delete and i%2 != 0: dt = f'cast({self.tdCom.date_time-1} as timestamp)' self.tdCom.sdelete_rows(tbname=self.ctb_name, start_ts=dt) @@ -166,6 +169,7 @@ class TDTestCase: self.tdCom.sdelete_rows(tbname=self.tb_name, start_ts=self.tdCom.time_cast(self.tdCom.record_history_ts, "-")) if self.tdCom.subtable: + group_id = self.tdCom.get_group_id_from_stb(f'{self.ctb_name}_output') tdSql.query(f'select * from {self.ctb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -182,11 +186,11 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.ctb_name}_{self.tdCom.subtable_prefix}{self.ctb_name}{self.tdCom.subtable_suffix}_{self.ctb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 tdSql.checkEqual(tdSql.queryResult[0][0] > 0, True) if subtable is not None else tdSql.checkEqual(tdSql.queryResult[0][0] >= 0, True) - + group_id = self.tdCom.get_group_id_from_stb(f'{self.tb_name}_output') tdSql.query(f'select * from {self.tb_name}') ptn_counter = 0 for c1_value in tdSql.queryResult: @@ -203,7 +207,7 @@ class TDTestCase: tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{partition_elm_alias}{self.tdCom.subtable_suffix}') tdSql.query(f'select count(*) from `{tbname}`') elif partition == "tbname" and ptn_counter == 0: - tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}') + tbname = self.tdCom.get_subtable_wait(f'{self.tb_name}_{self.tdCom.subtable_prefix}{self.tb_name}{self.tdCom.subtable_suffix}_{self.tb_name}_output_{group_id}') tdSql.query(f'select count(*) from `{tbname}`') ptn_counter += 1 diff --git a/tests/system-test/8-stream/stream_multi_agg.py b/tests/system-test/8-stream/stream_multi_agg.py index 3532825493..acb80f528b 100644 --- a/tests/system-test/8-stream/stream_multi_agg.py +++ b/tests/system-test/8-stream/stream_multi_agg.py @@ -39,7 +39,7 @@ class TDTestCase: os.system("nohup taosBenchmark -y -B 1 -t 40 -S 1000 -n 10 -i 1000 -v 5 > /dev/null 2>&1 &") time.sleep(10) - tdSql.query("use test") + tdSql.execute("use test", queryTimes=100) tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") tdLog.debug("========create stream and insert data ok========") time.sleep(15) @@ -66,7 +66,7 @@ class TDTestCase: os.system("taosBenchmark -d db -t 20 -v 6 -n 1000 -y > /dev/null 2>&1") # create stream - tdSql.execute("use db") + tdSql.execute("use db", queryTimes=100) tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True) time.sleep(5) diff --git a/tests/system-test/8-stream/window_close_session.py b/tests/system-test/8-stream/window_close_session.py index ddd366b45a..f9f6baab75 100644 --- a/tests/system-test/8-stream/window_close_session.py +++ b/tests/system-test/8-stream/window_close_session.py @@ -45,6 +45,9 @@ class TDTestCase: for ts_value in [self.tdCom.date_time, window_close_ts-1]: self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) + if i == 0 and fill_history_value is not None: + for tbname in [self.ctb_stream_des_table, self.tb_stream_des_table]: + tdSql.query(f'select count(*) from {tbname}', count_expected_res=1) if self.tdCom.update and i%2 == 0: self.tdCom.sinsert_rows(tbname=self.ctb_name, ts_value=ts_value) self.tdCom.sinsert_rows(tbname=self.tb_name, ts_value=ts_value) diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 2837c817be..060717c20e 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -89,8 +89,8 @@ else export LD_PRELOAD="$(realpath "$(gcc -print-file-name=libasan.so)") $(realpath "$(gcc -print-file-name=libstdc++.so)")" echo "Preload AsanSo:" $? - $* -a 2>$AsanFile - + $* -a 2> $AsanFile + cat $AsanFile unset LD_PRELOAD for ((i = 1; i <= 20; i++)); do AsanFileLen=$(cat $AsanFile | wc -l) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 188abb4b58..a16a03d30a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -140,8 +140,8 @@ ELSE () BUILD_COMMAND COMMAND set CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client COMMAND set CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib - COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND go build -a -o taosadapter.exe -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND go build -a -o taosadapter-debug.exe -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Comparessing taosadapter.exe" @@ -167,8 +167,8 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin @@ -192,19 +192,19 @@ ELSE () PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-s -w -X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" - COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_DATE}'" + COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" +# COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../include/client CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -a -o taosadapter-debug -ldflags "-X 'github.com/taosdata/taosadapter/v3/version.Version=${taos_version}' -X 'github.com/taosdata/taosadapter/v3/version.CommitID=${taosadapter_commit_sha1}' -X 'github.com/taosdata/taosadapter/v3/version.BuildInfo=${TD_VER_OSTYPE}-${TD_VER_CPUTYPE} ${TD_VER_DATE}'" INSTALL_COMMAND - COMMAND cmake -E echo "Comparessing taosadapter.exe" - COMMAND upx taosadapter || : +# COMMAND cmake -E echo "Comparessing taosadapter.exe" +# COMMAND upx taosadapter || : COMMAND cmake -E echo "Copy taosadapter" COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E echo "Copy taosadapter.toml" COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./taosadapter.service ${CMAKE_BINARY_DIR}/test/cfg/ - COMMAND cmake -E echo "Copy taosadapter-debug" - COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin +# COMMAND cmake -E echo "Copy taosadapter-debug" +# COMMAND cmake -E copy taosadapter-debug ${CMAKE_BINARY_DIR}/build/bin ) ENDIF () ENDIF () diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 434b5e4b61..f48b19a287 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -435,12 +435,12 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.promptSize = strlen(shell.info.promptHeader); #ifdef TD_ENTERPRISE snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), - "version: %s compatible_version: %s\ngitinfo: %s\ngitinfoOfInternal: %s\nbuildInfo: %s", version, + "%s\ntaos version: %s compatible_version: %s\ngit: %s\ngit: %s\nbuild: %s", TD_PRODUCT_NAME, version, compatible_version, gitinfo, gitinfoOfInternal, buildinfo); #else snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), - "version: %s compatible_version: %s\ngitinfo: %s\nbuildInfo: %s", version, compatible_version, gitinfo, - buildinfo); + "%s\ntaos version: %s compatible_version: %s\ngit: %s\nbuild: %s", TD_PRODUCT_NAME, version, + compatible_version, gitinfo, buildinfo); #endif #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1d1050c1ec..e04566aadb 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1189,12 +1189,11 @@ bool shellGetGrantInfo(char* buf) { fprintf(stderr, "\r\nFailed to get grant information from server. Abort.\r\n"); exit(0); } - - char serverVersion[32] = {0}; + char serverVersion[64] = {0}; char expiretime[32] = {0}; char expired[32] = {0}; - memcpy(serverVersion, row[0], fields[0].bytes); + tstrncpy(serverVersion, row[0], 64); memcpy(expiretime, row[1], fields[1].bytes); memcpy(expired, row[2], fields[2].bytes); @@ -1202,10 +1201,10 @@ bool shellGetGrantInfo(char* buf) { community = true; } else if (strcmp(expiretime, "unlimited") == 0) { community = false; - sprintf(buf, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo); + sprintf(buf, "Server is %s, %s and will never expire.\r\n", serverVersion, sinfo); } else { community = false; - sprintf(buf, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo, + sprintf(buf, "Server is %s, %s and will expire at %s.\r\n", serverVersion, sinfo, expiretime); } diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 5e9d82e71e..f91b14b6d8 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -982,6 +982,9 @@ int smlProcess_18784_Test() { rowIndex++; } taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists db_18784"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -1187,6 +1190,9 @@ int sml_ts2164_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists line_test"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -1236,6 +1242,8 @@ int sml_ts3116_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); code = taos_errno(pRes); taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists ts3116"); + taos_free_result(pRes); taos_close(taos); return code; @@ -1449,6 +1457,13 @@ int sml_td24070_Test() { code = taos_errno(pRes); ASSERT(code == 0); taos_free_result(pRes); + + pRes = taos_query(taos, "drop database if exists td24070_write"); + taos_free_result(pRes); + + pRes = taos_query(taos, "drop database if exists td24070_read"); + taos_free_result(pRes); + taos_close(taos); // test table privilege @@ -1477,6 +1492,10 @@ int sml_td23881_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + + pRes = taos_query(taos, "drop database if exists line_23881"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -1691,6 +1710,8 @@ int sml_ts2385_Test() { code = taos_errno(pRes); taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists ts2385"); + taos_free_result(pRes); taos_close(taos); return code; @@ -1784,6 +1805,9 @@ int sml_td24559_Test() { } taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists td24559"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -1889,6 +1913,9 @@ int sml_td29691_Test() { ASSERT(code == TSDB_CODE_PAR_DUPLICATED_COLUMN); taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists td29691"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -1942,6 +1969,9 @@ int sml_td18789_Test() { } taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists td18789"); + taos_free_result(pRes); + taos_close(taos); return code; @@ -2047,6 +2077,9 @@ int sml_td29373_Test() { ASSERT(code == TSDB_CODE_SML_NOT_SUPPORT_PK); taos_free_result(pRes); + pRes = taos_query(taos, "drop database if exists td29373"); + taos_free_result(pRes); + taos_close(taos); return code; diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index a391093609..51d134a463 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -18,6 +18,7 @@ #include #include #include +#include "cJSON.h" #include "taos.h" #include "tmsg.h" #include "types.h" @@ -32,6 +33,7 @@ typedef struct { int srcVgroups; int dstVgroups; char dir[256]; + bool btMeta; } Config; Config g_conf = {0}; @@ -61,8 +63,23 @@ static void msg_process(TAOS_RES* msg) { if (result) { printf("meta result: %s\n", result); if (g_fp && strcmp(result, "") != 0) { - taosFprintfFile(g_fp, result); - taosFprintfFile(g_fp, "\n"); + // RES_TYPE__TMQ_BATCH_META + if ((*(int8_t*)msg) == 5) { + cJSON* pJson = cJSON_Parse(result); + cJSON* pJsonArray = cJSON_GetObjectItem(pJson, "metas"); + int32_t num = cJSON_GetArraySize(pJsonArray); + for (int32_t i = 0; i < num; i++) { + cJSON* pJsonItem = cJSON_GetArrayItem(pJsonArray, i); + char* itemStr = cJSON_PrintUnformatted(pJsonItem); + taosFprintfFile(g_fp, itemStr); + tmq_free_json_meta(itemStr); + taosFprintfFile(g_fp, "\n"); + } + cJSON_Delete(pJson); + } else { + taosFprintfFile(g_fp, result); + taosFprintfFile(g_fp, "\n"); + } } } tmq_free_json_meta(result); @@ -584,6 +601,10 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); } + if (g_conf.btMeta) { + tmq_conf_set(conf, "msg.enable.batchmeta", "true"); + } + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); @@ -1064,7 +1085,7 @@ void testConsumeExcluded(int topic_type) { char* topic = "create topic topic_excluded with meta as database db_taosx"; pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes)); + printf("failed to create topic topic_excluded1, reason:%s\n", taos_errstr(pRes)); taos_close(pConn); return; } @@ -1073,7 +1094,7 @@ void testConsumeExcluded(int topic_type) { char* topic = "create topic topic_excluded as select * from stt"; pRes = taos_query(pConn, topic); if (taos_errno(pRes) != 0) { - printf("failed to create topic topic_excluded, reason:%s\n", taos_errstr(pRes)); + printf("failed to create topic topic_excluded2, reason:%s\n", taos_errstr(pRes)); taos_close(pConn); return; } @@ -1115,7 +1136,7 @@ void testConsumeExcluded(int topic_type) { assert(raw.raw_type != 2 && raw.raw_type != 4 && raw.raw_type != TDMT_VND_CREATE_STB && raw.raw_type != TDMT_VND_ALTER_STB && raw.raw_type != TDMT_VND_CREATE_TABLE && raw.raw_type != TDMT_VND_ALTER_TABLE && raw.raw_type != TDMT_VND_DELETE); - assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE); + assert(raw.raw_type == TDMT_VND_DROP_STB || raw.raw_type == TDMT_VND_DROP_TABLE || raw.raw_type == 5); } else if (topic_type == 2) { assert(0); } @@ -1138,6 +1159,7 @@ void testConsumeExcluded(int topic_type) { taos_close(pConn); return; } + taos_close(pConn); taos_free_result(pRes); } @@ -1167,6 +1189,8 @@ int main(int argc, char* argv[]) { g_conf.subTable = true; } else if (strcmp(argv[i], "-onlymeta") == 0) { g_conf.meta = 1; + } else if (strcmp(argv[i], "-bt") == 0) { + g_conf.btMeta = true; } } diff --git a/utils/test/c/write_raw_block_test.c b/utils/test/c/write_raw_block_test.c index 162ecd229c..0f123fb560 100644 --- a/utils/test/c/write_raw_block_test.c +++ b/utils/test/c/write_raw_block_test.c @@ -78,7 +78,7 @@ int buildStable(TAOS* pConn) { } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ntba values(now,'hello')"); + pRes = taos_query(pConn, "insert into ntba values(now + 1s,'hello')"); if (taos_errno(pRes) != 0) { printf("failed to insert table ntba, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index 645cf0e6fe..dcf5d6ab12 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -28,7 +28,7 @@ extern bool simExecSuccess; int32_t simInitCfg() { taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); - taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true); + taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg(); tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX);