From 803ed61ca00bc4506d05fe8a9c7e5248f66a2a9c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Mon, 2 Dec 2024 09:29:55 +0800 Subject: [PATCH 1/3] ci(stream):add long time test for stream --- tests/parallel_test/longtimeruning_cases.task | 3 +++ tests/script/tsim/stream/streamTwaInterpFwcCheckpoint.sim | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/longtimeruning_cases.task b/tests/parallel_test/longtimeruning_cases.task index 1ec234c266..1034771159 100644 --- a/tests/parallel_test/longtimeruning_cases.task +++ b/tests/parallel_test/longtimeruning_cases.task @@ -13,6 +13,9 @@ #tsim test #,,y,script,./test.sh -f tsim/query/timeline.sim +,,y,script,./test.sh -f tsim/stream/streamTwaInterpFwcCheckpoint.sim +,,y,script,./test.sh -f tsim/stream/streamFwcIntervalFill.sim +,,y,script,./test.sh -f tsim/stream/streamTwaFwcInterval.sim #docs-examples test #,,n,docs-examples-test,bash c.sh diff --git a/tests/script/tsim/stream/streamTwaInterpFwcCheckpoint.sim b/tests/script/tsim/stream/streamTwaInterpFwcCheckpoint.sim index 26be5018c1..8d30e1c1d3 100644 --- a/tests/script/tsim/stream/streamTwaInterpFwcCheckpoint.sim +++ b/tests/script/tsim/stream/streamTwaInterpFwcCheckpoint.sim @@ -110,10 +110,10 @@ print =====streamt4_rows=$streamt4_rows $loop_count = 0 loop1: -sleep 2000 +sleep 6000 $loop_count = $loop_count + 1 -if $loop_count == 20 then +if $loop_count == 100 then return -1 endi @@ -154,7 +154,7 @@ sql insert into t1 values(now + 3000a,10,10,10); $loop_count = 0 loop2: -sleep 2000 +sleep 6000 $loop_count = $loop_count + 1 if $loop_count == 20 then From cdecfaf59fc1aadc5a671a41fd23066437f649f0 Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Mon, 2 Dec 2024 15:58:44 +0800 Subject: [PATCH 2/3] Update 01-subscription.md add message that lost --- docs/zh/06-advanced/01-subscription.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/zh/06-advanced/01-subscription.md b/docs/zh/06-advanced/01-subscription.md index c44bfdbf6f..5d458b186c 100644 --- a/docs/zh/06-advanced/01-subscription.md +++ b/docs/zh/06-advanced/01-subscription.md @@ -12,6 +12,8 @@ toc_max_heading_level: 4 为实现上述功能,TDengine 会为预写数据日志(Write-Ahead Logging,WAL)文件自动创建索引,以支持快速随机访问,并提供了灵活可配置的文件切换与保留机制。用户可以根据需求指定 WAL 文件的保留时间和大小。通过这些方法,WAL 被改造成一个保留事件到达顺序的、可持久化的存储引擎。对于以主题形式创建的查询,TDengine 将从 WAL 读取数据。在消费过程中,TDengine 根据当前消费进度从 WAL 直接读取数据,并使用统一的查询引擎实现过滤、变换等操作,然后将数据推送给消费者。 +从 3.2.0.0 版本开始,数据订阅支持 vnode 迁移和分裂。 由于数据订阅依赖 wal文件,而在 vnode 迁移和分裂的过程中,wal 并不会同步过去,所以迁移或分裂后,之前没消费完的 wal数据后消费不到。所以请保证之前把数据全部消费完后,再进行 vnode 迁移或分裂,否则,消费会丢失数据。 + ## 主题类型 TDengine 使用 SQL 创建的主题共有 3 种类型,下面分别介绍。 @@ -129,4 +131,4 @@ TDengine 的数据订阅功能支持回放(replay)功能,允许用户按 使用数据订阅的回放功能时需要注意如下几项。 - 数据订阅的回放功能仅查询订阅支持数据回放,超级表和库订阅不支持回放。 - 回放不支持进度保存。 -- 因为数据回放本身需要处理时间,所以回放的精度存在几十毫秒的误差。 \ No newline at end of file +- 因为数据回放本身需要处理时间,所以回放的精度存在几十毫秒的误差。 From efe943d951c126d3744504e31ed3c823ea5c2b0c Mon Sep 17 00:00:00 2001 From: WANG MINGMING Date: Mon, 2 Dec 2024 17:28:37 +0800 Subject: [PATCH 3/3] Update 01-subscription.md --- docs/en/06-advanced/01-subscription.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/06-advanced/01-subscription.md b/docs/en/06-advanced/01-subscription.md index af8094c640..2668dae19b 100644 --- a/docs/en/06-advanced/01-subscription.md +++ b/docs/en/06-advanced/01-subscription.md @@ -9,6 +9,8 @@ TDengine's message queue provides an ACK (Acknowledgment) mechanism to ensure at To achieve the above functionality, TDengine automatically creates indexes for the Write-Ahead Logging (WAL) files to support fast random access and provides a flexible and configurable file switching and retention mechanism. Users can specify the retention time and size of the WAL files according to their needs. Through these methods, the WAL is transformed into a persistent storage engine that preserves the order of event arrival. For queries created in the form of topics, TDengine will read data from the WAL. During consumption, TDengine reads data directly from the WAL based on the current consumption progress and uses a unified query engine to perform filtering, transformation, and other operations before pushing the data to consumers. +Starting from version 3.2.0.0, data subscription supports vnode migration and splitting. Due to the dependence of data subscription on wal files, wal does not synchronize during vnode migration and splitting. Therefore, after migration or splitting, wal data that has not been consumed before cannot be consumed. So please ensure that all data has been consumed before proceeding with vnode migration or splitting, otherwise data loss may occur during consumption. + ## Topics A topic can be a query, a supertable, or a database. You can filter by tag, table name, column, or expression and perform scalar operations. Note that data aggregation and time windows are not supported. The data granularity is determined by the SQL statement that defines the topic, and data filtering and preprocessing are automatically handled by TDengine.