Merge branch '3.0' of github.com:taosdata/TDengine into szhou/continue-coing

docs/en/07-develop/03-insert-data/30-influxdb-line.mdx

@@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
 - All the data in `tag_set` will be converted to NCHAR type automatically
 - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double
 - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h)
-- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
+- The rule of table name
+  - The child table name is created automatically in a rule to guarantee its uniqueness.
+  - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4`
+  - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
 - It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0)
 
 :::

docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx

@@ -33,7 +33,10 @@ For example:
 meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
 ```
 
-- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
+- The rule of table name
+  - The child table name is created automatically in a rule to guarantee its uniqueness.
+  - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4`
+  - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
 
   Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details.
 

docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx

@@ -48,7 +48,10 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http
 :::note
 
 - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type.
-- The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
+- The rule of table name
+  - The child table name is created automatically in a rule to guarantee its uniqueness.
+  - You can configure `smlAutoChildTableNameDelimiter` in taos.cfg to specify a delimiter between tag values as the table names. For example, you set `smlAutoChildTableNameDelimiter=-` in taos.cfg, when you insert `st,t0=cpu1,t1=4 c1=3 1626006833639000000`, the child table will be `cpu1-4`
+  - You can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored
 
 :::
 

docs/en/07-develop/07-tmq.mdx

@@ -356,6 +356,7 @@ You configure the following parameters when creating a consumer:
 |      `enable.auto.commit`      | boolean | Commit automatically; true: user application doesn't need to explicitly commit; false: user application need to handle commit by itself                                           | Default value is true                  |
 |   `auto.commit.interval.ms`    | integer | Interval for automatic commits, in milliseconds                           |
 |     `msg.with.table.name`      | boolean | Specify whether to deserialize table names from messages. Not applicable if subscribe to a column (tbname can be written as a column in the subquery statement during column subscriptions) (This parameter has been deprecated since version 3.2.0.0 and remains true)                                 | default value: false
+|     `enable.replay`            | boolean | Specify whether data replay function enabled or not                   |default value: false |
 
 The method of specifying these parameters depends on the language used:
 
@@ -371,7 +372,7 @@ tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
 tmq_conf_set(conf, "group.id", "cgrpName");
 tmq_conf_set(conf, "td.connect.user", "root");
 tmq_conf_set(conf, "td.connect.pass", "taosdata");
-tmq_conf_set(conf, "auto.offset.reset", "earliest");
+tmq_conf_set(conf, "auto.offset.reset", "latest");
 tmq_conf_set(conf, "msg.with.table.name", "true");
 tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
 
@@ -401,7 +402,7 @@ properties.setProperty("group.id", "cgrpName");
 properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
 properties.setProperty("td.connect.user", "root");
 properties.setProperty("td.connect.pass", "taosdata");
-properties.setProperty("auto.offset.reset", "earliest");
+properties.setProperty("auto.offset.reset", "latest");
 properties.setProperty("msg.with.table.name", "true");
 properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
 
@@ -421,7 +422,7 @@ public class MetersDeserializer extends ReferenceDeserializer<Meters> {
 ```go
 conf := &tmq.ConfigMap{
  "group.id":                     "test",
- "auto.offset.reset":            "earliest",
+ "auto.offset.reset":            "latest",
  "td.connect.ip":                "127.0.0.1",
  "td.connect.user":              "root",
  "td.connect.pass":              "taosdata",
@@ -441,7 +442,7 @@ consumer, err := NewConsumer(conf)
 let mut dsn: Dsn = "taos://".parse()?;
 dsn.set("group.id", "group1");
 dsn.set("client.id", "test");
-dsn.set("auto.offset.reset", "earliest");
+dsn.set("auto.offset.reset", "latest");
 
 let tmq = TmqBuilder::from_dsn(dsn)?;
 
@@ -467,7 +468,7 @@ consumer = Consumer(
         "td.connect.ip": "127.0.0.1",
         "td.connect.user": "root",
         "td.connect.pass": "taosdata",
-        "auto.offset.reset": "earliest",
+        "auto.offset.reset": "latest",
         "msg.with.table.name": "true",
     }
 )
@@ -487,7 +488,7 @@ let consumer = taos.consumer({
   'group.id': 'tg2',
   'td.connect.user': 'root',
   'td.connect.pass': 'taosdata',
-  'auto.offset.reset','earliest',
+  'auto.offset.reset','latest',
   'msg.with.table.name': 'true',
   'td.connect.ip','127.0.0.1',
   'td.connect.port','6030'  
@@ -510,7 +511,7 @@ var cfg = new ConsumerConfig
     GourpId = "TDengine-TMQ-C#",
     TDConnectUser = "root",
     TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "earliest"
+    AutoOffsetReset = "latest"
     MsgWithTableName = "true",
     TDConnectIp = "127.0.0.1",
     TDConnectPort = "6030"
@@ -526,6 +527,24 @@ var consumer = new ConsumerBuilder(cfg).Build();
 
 A consumer group is automatically created when multiple consumers are configured with the same consumer group ID.
 
+Data replay function description:
+- Subscription adds replay function, which replays according to the time of data writing.
+  For example, writing three pieces of data at the following time.
+  ```sql
+    2023/09/22 00:00:00.000
+    2023/09/22 00:00:05.000
+    2023/09/22 00:00:08.000
+    ```
+  After subscribing to the first data for 5 seconds, the second data is returned, and after obtaining the second data for 3 seconds, the third data is returned.
+- Only column subscriptions support data replay.
+  - Replay needs to ensure an independent timeline
+  - If it is a sub table subscription or a normal table subscription, only one vnode has data, ensuring a timeline.
+  - If subscribing to a super table, it is necessary to ensure that the DB has only one vnode, otherwise an error will be reported (because the data subscribed to on multiple vnodes is not on the same timeline).
+- Super table and database subscriptions do not support replay
+- Add the enable.replay parameter. True indicates that the subscription replay function is enabled, while false indicates that the subscription replay function is not enabled by default.
+- Replay does not support progress saving, so when the replay parameter enable, auto commit will automatically close.
+- Due to the processing time required for data replay, there is an error of tens of milliseconds in the accuracy of replay.
+
 ## Subscribe to a Topic
 
 A single consumer can subscribe to multiple topics.

docs/en/14-reference/03-connector/04-java.mdx

@@ -1093,7 +1093,7 @@ TaosConsumer consumer = new TaosConsumer<>(config);
 - httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type.
 - messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type.
 - httpPoolSize: Maximum number of concurrent requests on the a connection。It only takes effect when using WebSocket type.
-- For more information, see [Consumer Parameters](../../../develop/tmq).
+- For more information, see [Consumer Parameters](../../../develop/tmq). Note that the default value of auto.offset.reset in data subscription on the TDengine server has changed since version 3.2.0.0.
 
 #### Subscribe to consume data
 
@@ -1193,7 +1193,7 @@ public abstract class ConsumerLoop {
         config.setProperty("bootstrap.servers", "localhost:6030");
         config.setProperty("td.connect.user", "root");
         config.setProperty("td.connect.pass", "taosdata");
-        config.setProperty("auto.offset.reset", "earliest");
+        config.setProperty("auto.offset.reset", "latest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
         config.setProperty("auto.commit.interval.ms", "1000");
@@ -1276,7 +1276,7 @@ public abstract class ConsumerLoop {
         config.setProperty("bootstrap.servers", "localhost:6041");
         config.setProperty("td.connect.user", "root");
         config.setProperty("td.connect.pass", "taosdata");
-        config.setProperty("auto.offset.reset", "earliest");
+        config.setProperty("auto.offset.reset", "latest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
         config.setProperty("auto.commit.interval.ms", "1000");

docs/en/14-reference/03-connector/05-go.mdx

@@ -794,7 +794,7 @@ The TDengine Go Connector supports subscription functionality with the following
 ```go
     consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{
         "group.id":                     "test",
-        "auto.offset.reset":            "earliest",
+        "auto.offset.reset":            "latest",
         "td.connect.ip":                "127.0.0.1",
         "td.connect.user":              "root",
         "td.connect.pass":              "taosdata",
@@ -870,6 +870,7 @@ package main
 import (
     "fmt"
     "os"
+    "time"
 
     "github.com/taosdata/driver-go/v3/af"
     "github.com/taosdata/driver-go/v3/af/tmq"
@@ -890,19 +891,16 @@ func main() {
     if err != nil {
         panic(err)
     }
-    if err != nil {
-        panic(err)
-    }
     consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{
-        "group.id":                     "test",
+        "group.id":            "test",
-        "auto.offset.reset":            "earliest",
+        "auto.offset.reset":   "latest",
-        "td.connect.ip":                "127.0.0.1",
+        "td.connect.ip":       "127.0.0.1",
-        "td.connect.user":              "root",
+        "td.connect.user":     "root",
-        "td.connect.pass":              "taosdata",
+        "td.connect.pass":     "taosdata",
-        "td.connect.port":              "6030",
+        "td.connect.port":     "6030",
-        "client.id":                    "test_tmq_client",
+        "client.id":           "test_tmq_client",
-        "enable.auto.commit":           "false",
+        "enable.auto.commit":  "false",
-        "msg.with.table.name":          "true",
+        "msg.with.table.name": "true",
     })
     if err != nil {
         panic(err)
@@ -915,10 +913,16 @@ func main() {
     if err != nil {
         panic(err)
     }
-    _, err = db.Exec("insert into example_tmq.t1 values(now,1)")
+    go func() {
-    if err != nil {
+        for {
-        panic(err)
+            _, err = db.Exec("insert into example_tmq.t1 values(now,1)")
-    }
+            if err != nil {
+                panic(err)
+            }
+            time.Sleep(time.Millisecond * 100)
+        }
+    }()
+
     for i := 0; i < 5; i++ {
         ev := consumer.Poll(500)
         if ev != nil {
@@ -972,6 +976,7 @@ package main
 import (
     "database/sql"
     "fmt"
+    "time"
 
     "github.com/taosdata/driver-go/v3/common"
     tmqcommon "github.com/taosdata/driver-go/v3/common/tmq"
@@ -995,7 +1000,7 @@ func main() {
         "td.connect.pass":       "taosdata",
         "group.id":              "example",
         "client.id":             "example_consumer",
-        "auto.offset.reset":     "earliest",
+        "auto.offset.reset":     "latest",
     })
     if err != nil {
         panic(err)
@@ -1004,29 +1009,34 @@ func main() {
     if err != nil {
         panic(err)
     }
+
+    _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," +
+        "c1 bool," +
+        "c2 tinyint," +
+        "c3 smallint," +
+        "c4 int," +
+        "c5 bigint," +
+        "c6 tinyint unsigned," +
+        "c7 smallint unsigned," +
+        "c8 int unsigned," +
+        "c9 bigint unsigned," +
+        "c10 float," +
+        "c11 double," +
+        "c12 binary(20)," +
+        "c13 nchar(20)" +
+        ")")
+    if err != nil {
+        panic(err)
+    }
     go func() {
-        _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," +
+        for {
-            "c1 bool," +
+            _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')")
-            "c2 tinyint," +
+            if err != nil {
-            "c3 smallint," +
+                panic(err)
-            "c4 int," +
+            }
-            "c5 bigint," +
+            time.Sleep(time.Millisecond * 100)
-            "c6 tinyint unsigned," +
-            "c7 smallint unsigned," +
-            "c8 int unsigned," +
-            "c9 bigint unsigned," +
-            "c10 float," +
-            "c11 double," +
-            "c12 binary(20)," +
-            "c13 nchar(20)" +
-            ")")
-        if err != nil {
-            panic(err)
-        }
-        _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')")
-        if err != nil {
-            panic(err)
         }
+
     }()
     for i := 0; i < 5; i++ {
         ev := consumer.Poll(500)

docs/en/14-reference/03-connector/06-rust.mdx

@@ -442,7 +442,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m
 
 - `group.id`: Within a consumer group, load balancing is implemented by consuming messages on an at-least-once basis.
 - `client.id`: Subscriber client ID.
-- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default is earliest. Note: This parameter is set per consumer group.
+- `auto.offset.reset`: Initial point of subscription. *earliest* subscribes from the beginning, and *latest* subscribes from the newest message. The default value varies depending on the TDengine version. For details, see [Data Subscription](https://docs.tdengine.com/develop/tmq/). Note: This parameter is set per consumer group.
 - `enable.auto.commit`: Automatically commits. This can be enabled when data consistency is not essential.
 - `auto.commit.interval.ms`: Interval for automatic commits.
 

docs/en/14-reference/12-config/index.md

@@ -652,6 +652,15 @@ The charset that takes effect is UTF-8.
 | Type          | String                                     |
 | Default Value | None                                       |
 
+### smlAutoChildTableNameDelimiter
+
+| Attribute     | Description                                |
+| ------------- | ------------------------------------------ |
+| Applicable    | Client only                                |
+| Meaning       | Delimiter between tags as table name|
+| Type          | String                                     |
+| Default Value | None                                       |
+
 ### smlTagName
 
 | Attribute     | Description                                                   |

docs/en/14-reference/13-schemaless/13-schemaless.md

@@ -93,6 +93,8 @@ Note that tag_key1, tag_key2 are not the original order of the tags entered by t
 The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has.
 :::
 
+If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority.
+You can configure smlAutoChildTableNameDelimiter in taos.cfg, for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4.
 You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored.
 
 2. If the super table obtained by parsing the line protocol does not exist, this super table is created.

docs/examples/go/sub/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/taosdata/driver-go/v3/af"
 	"github.com/taosdata/driver-go/v3/af/tmq"
@@ -27,15 +28,15 @@ func main() {
 		panic(err)
 	}
 	consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{
-		"group.id":                     "test",
+		"group.id":            "test",
-		"auto.offset.reset":            "earliest",
+		"auto.offset.reset":   "latest",
-		"td.connect.ip":                "127.0.0.1",
+		"td.connect.ip":       "127.0.0.1",
-		"td.connect.user":              "root",
+		"td.connect.user":     "root",
-		"td.connect.pass":              "taosdata",
+		"td.connect.pass":     "taosdata",
-		"td.connect.port":              "6030",
+		"td.connect.port":     "6030",
-		"client.id":                    "test_tmq_client",
+		"client.id":           "test_tmq_client",
-		"enable.auto.commit":           "false",
+		"enable.auto.commit":  "false",
-		"msg.with.table.name":          "true",
+		"msg.with.table.name": "true",
 	})
 	if err != nil {
 		panic(err)
@@ -48,12 +49,17 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	_, err = db.Exec("insert into example_tmq.t1 values(now,1)")
+	go func() {
-	if err != nil {
+		for {
-		panic(err)
+			_, err = db.Exec("insert into example_tmq.t1 values(now,1)")
-	}
+			if err != nil {
+				panic(err)
+			}
+			time.Sleep(time.Microsecond * 100)
+		}
+	}()
 	for i := 0; i < 5; i++ {
-		ev := consumer.Poll(0)
+		ev := consumer.Poll(500)
 		if ev != nil {
 			switch e := ev.(type) {
 			case *tmqcommon.DataMessage:

docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java

@@ -66,7 +66,6 @@ public class SubscribeDemo {
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
                     "com.taos.example.MetersDeserializer");
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
-            properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
 
             // poll data
             try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {

docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java

@@ -66,7 +66,6 @@ public class WebsocketSubscribeDemo {
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER,
                     "com.taos.example.MetersDeserializer");
             properties.setProperty(TMQConstants.VALUE_DESERIALIZER_ENCODING, "UTF-8");
-            properties.setProperty(TMQConstants.EXPERIMENTAL_SNAPSHOT_ENABLE, "true");
 
             // poll data
             try (TaosConsumer<Meters> consumer = new TaosConsumer<>(properties)) {

docs/examples/python/tmq_assignment_example.py

@@ -23,9 +23,6 @@ def taos_get_assignment_and_seek_demo():
     consumer = Consumer(
         {
             "group.id": "0",
-            # should disable snapshot,
-            # otherwise it will cause invalid params error
-            "experimental.snapshot.enable": "false",
         }
     )
     consumer.subscribe(["tmq_assignment_demo_topic"])

docs/examples/python/tmq_websocket_assgnment_example.py

@@ -21,9 +21,6 @@ def taosws_get_assignment_and_seek_demo():
     prepare()
     consumer = taosws.Consumer(conf={
         "td.connect.websocket.scheme": "ws",
-        # should disable snapshot,
-        # otherwise it will cause invalid params error
-        "experimental.snapshot.enable": "false",
         "group.id": "0",
     })
     consumer.subscribe(["tmq_assignment_demo_topic"])

docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx

@@ -38,7 +38,10 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
 - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理
 - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度，支持从小时到纳秒的 6 种时间精度
 - 为了提高写入的效率，默认假设同一个超级表中 field_set 的顺序是一样的（第一条数据包含所有的 field，后面的数据按照这个顺序），如果顺序不一样，需要配置参数 smlDataFormat 为 false，否则，数据写入按照相同顺序写入，库中数据会异常。（3.0.1.3 之后的版本 smlDataFormat 默认为 false，从3.0.3.0开始，该配置废弃） [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
-- 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
+- 子表名生成规则
+  - 默认产生的子表名是根据规则生成的唯一 ID 值。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
 
 :::
 

docs/zh/07-develop/03-insert-data/40-opentsdb-telnet.mdx

@@ -31,8 +31,11 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB
 ```txt
 meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3
 ```
+- 子表名生成规则
+  - 默认产生的子表名是根据规则生成的唯一 ID 值。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
 
-- 默认生产的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
   参考 [OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。
 
 ## 示例代码

docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx

@@ -47,7 +47,10 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据
 :::note
 
 - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。
-- 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
+- 子表名生成规则
+  - 默认产生的子表名是根据规则生成的唯一 ID 值。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlAutoChildTableNameDelimiter 参数来指定连接标签之间的分隔符，连接起来后作为子表名。举例如下：配置 smlAutoChildTableNameDelimiter=-, 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1-4。
+  - 用户也可以通过在client端的 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议)
 
 :::
 

docs/zh/07-develop/07-tmq.mdx

@@ -355,6 +355,7 @@ CREATE TOPIC topic_name [with meta] AS DATABASE db_name;
 |      `enable.auto.commit`      | boolean | 是否启用消费位点自动提交，true: 自动提交，客户端应用无需commit；false：客户端应用需要自行commit     | 默认值为 true                   |
 |   `auto.commit.interval.ms`    | integer | 消费记录自动提交消费位点时间间隔，单位为毫秒           | 默认值为 5000                                |
 |     `msg.with.table.name`      | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句）（从3.2.0.0版本该参数废弃，恒为true）               |默认关闭 |
+|     `enable.replay`            | boolean | 是否开启数据回放功能               |默认关闭 |
 
 对于不同编程语言，其设置方式如下：
 
@@ -370,7 +371,7 @@ tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
 tmq_conf_set(conf, "group.id", "cgrpName");
 tmq_conf_set(conf, "td.connect.user", "root");
 tmq_conf_set(conf, "td.connect.pass", "taosdata");
-tmq_conf_set(conf, "auto.offset.reset", "earliest");
+tmq_conf_set(conf, "auto.offset.reset", "latest");
 tmq_conf_set(conf, "msg.with.table.name", "true");
 tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL);
 
@@ -400,7 +401,7 @@ properties.setProperty("group.id", "cgrpName");
 properties.setProperty("bootstrap.servers", "127.0.0.1:6030");
 properties.setProperty("td.connect.user", "root");
 properties.setProperty("td.connect.pass", "taosdata");
-properties.setProperty("auto.offset.reset", "earliest");
+properties.setProperty("auto.offset.reset", "latest");
 properties.setProperty("msg.with.table.name", "true");
 properties.setProperty("value.deserializer", "com.taos.example.MetersDeserializer");
 
@@ -420,7 +421,7 @@ public class MetersDeserializer extends ReferenceDeserializer<Meters> {
 ```go
 conf := &tmq.ConfigMap{
  "group.id":                     "test",
- "auto.offset.reset":            "earliest",
+ "auto.offset.reset":            "latest",
  "td.connect.ip":                "127.0.0.1",
  "td.connect.user":              "root",
  "td.connect.pass":              "taosdata",
@@ -440,7 +441,7 @@ consumer, err := NewConsumer(conf)
 let mut dsn: Dsn = "taos://".parse()?;
 dsn.set("group.id", "group1");
 dsn.set("client.id", "test");
-dsn.set("auto.offset.reset", "earliest");
+dsn.set("auto.offset.reset", "latest");
 
 let tmq = TmqBuilder::from_dsn(dsn)?;
 
@@ -468,7 +469,7 @@ consumer = Consumer(
         "td.connect.ip": "127.0.0.1",
         "td.connect.user": "root",
         "td.connect.pass": "taosdata",
-        "auto.offset.reset": "earliest",
+        "auto.offset.reset": "latest",
         "msg.with.table.name": "true",
     }
 )
@@ -488,7 +489,7 @@ let consumer = taos.consumer({
   'group.id': 'tg2',
   'td.connect.user': 'root',
   'td.connect.pass': 'taosdata',
-  'auto.offset.reset','earliest',
+  'auto.offset.reset','latest',
   'msg.with.table.name': 'true',
   'td.connect.ip','127.0.0.1',
   'td.connect.port','6030'  
@@ -511,7 +512,7 @@ var cfg = new ConsumerConfig
     GourpId = "TDengine-TMQ-C#",
     TDConnectUser = "root",
     TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "earliest"
+    AutoOffsetReset = "latest"
     MsgWithTableName = "true",
     TDConnectIp = "127.0.0.1",
     TDConnectPort = "6030"
@@ -527,6 +528,24 @@ var consumer = new ConsumerBuilder(cfg).Build();
 
 上述配置中包括 consumer group ID，如果多个 consumer 指定的 consumer group ID 一样，则自动形成一个 consumer group，共享消费进度。
 
+数据回放功能说明：
+- 订阅增加 replay 功能，按照数据写入的时间回放。
+  比如，如下时间写入三条数据
+  ```sql
+    2023/09/22 00:00:00.000
+    2023/09/22 00:00:05.000
+    2023/09/22 00:00:08.000
+    ```
+  则订阅出第一条数据 5s 后返回第二条数据，获取第二条数据 3s 后返回第三条数据。
+- 仅列订阅支持数据回放
+  - 回放需要保证独立时间线
+  - 如果是子表订阅或者普通表订阅，只有一个vnode上有数据，保证是一个时间线
+  - 如果超级表订阅，则需保证该 DB 只有一个vnode，否则报错（因为多个vnode上订阅出的数据不在一个时间线上）
+- 超级表和库订阅不支持回放
+- 增加 enable.replay 参数，true表示开启订阅回放功能，false表示不开启订阅回放功能，默认不开启。
+- 回放不支持进度保存，所以回放参数 enable.replay = true 时，auto commit 自动关闭
+- 因为数据回放本身需要处理时间，所以回放的精度存在几十ms的误差
+
 ## 订阅 *topics*
 
 一个 consumer 支持同时订阅多个 topic。

docs/zh/08-connector/14-java.mdx

@@ -1095,7 +1095,8 @@ TaosConsumer consumer = new TaosConsumer<>(config);
 - httpConnectTimeout: 创建连接超时参数，单位 ms，默认为 5000 ms。仅在 WebSocket 连接下有效。
 - messageWaitTimeout: 数据传输超时参数，单位 ms，默认为 10000 ms。仅在 WebSocket 连接下有效。
 - httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。
-  其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group)
+  其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group)， 注意TDengine服务端自3.2.0.0版本开始消息订阅中的auto.offset.reset默认值发生变化。
+  
 
 #### 订阅消费数据
 
@@ -1193,7 +1194,7 @@ public abstract class ConsumerLoop {
         config.setProperty("bootstrap.servers", "localhost:6030");
         config.setProperty("td.connect.user", "root");
         config.setProperty("td.connect.pass", "taosdata");
-        config.setProperty("auto.offset.reset", "earliest");
+        config.setProperty("auto.offset.reset", "latest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
         config.setProperty("auto.commit.interval.ms", "1000");
@@ -1201,7 +1202,6 @@ public abstract class ConsumerLoop {
         config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
         config.setProperty("value.deserializer.encoding", "UTF-8");
-        config.setProperty("experimental.snapshot.enable", "true");
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");
@@ -1279,7 +1279,7 @@ public abstract class ConsumerLoop {
         config.setProperty("bootstrap.servers", "localhost:6041");
         config.setProperty("td.connect.user", "root");
         config.setProperty("td.connect.pass", "taosdata");
-        config.setProperty("auto.offset.reset", "earliest");
+        config.setProperty("auto.offset.reset", "latest");
         config.setProperty("msg.with.table.name", "true");
         config.setProperty("enable.auto.commit", "true");
         config.setProperty("auto.commit.interval.ms", "1000");
@@ -1287,7 +1287,6 @@ public abstract class ConsumerLoop {
         config.setProperty("client.id", "1");
         config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer");
         config.setProperty("value.deserializer.encoding", "UTF-8");
-        config.setProperty("experimental.snapshot.enable", "true");
 
         this.consumer = new TaosConsumer<>(config);
         this.topics = Collections.singletonList("topic_speed");

docs/zh/08-connector/20-go.mdx

@@ -797,7 +797,7 @@ TDengine Go 连接器支持订阅功能，应用 API 如下：
 ```go
     consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{
         "group.id":                     "test",
-        "auto.offset.reset":            "earliest",
+        "auto.offset.reset":            "latest",
         "td.connect.ip":                "127.0.0.1",
         "td.connect.user":              "root",
         "td.connect.pass":              "taosdata",
@@ -873,6 +873,7 @@ package main
 import (
     "fmt"
     "os"
+    "time"
 
     "github.com/taosdata/driver-go/v3/af"
     "github.com/taosdata/driver-go/v3/af/tmq"
@@ -893,19 +894,16 @@ func main() {
     if err != nil {
         panic(err)
     }
-    if err != nil {
-        panic(err)
-    }
     consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{
-        "group.id":                     "test",
+        "group.id":            "test",
-        "auto.offset.reset":            "earliest",
+        "auto.offset.reset":   "latest",
-        "td.connect.ip":                "127.0.0.1",
+        "td.connect.ip":       "127.0.0.1",
-        "td.connect.user":              "root",
+        "td.connect.user":     "root",
-        "td.connect.pass":              "taosdata",
+        "td.connect.pass":     "taosdata",
-        "td.connect.port":              "6030",
+        "td.connect.port":     "6030",
-        "client.id":                    "test_tmq_client",
+        "client.id":           "test_tmq_client",
-        "enable.auto.commit":           "false",
+        "enable.auto.commit":  "false",
-        "msg.with.table.name":          "true",
+        "msg.with.table.name": "true",
     })
     if err != nil {
         panic(err)
@@ -918,10 +916,16 @@ func main() {
     if err != nil {
         panic(err)
     }
-    _, err = db.Exec("insert into example_tmq.t1 values(now,1)")
+    go func() {
-    if err != nil {
+        for {
-        panic(err)
+            _, err = db.Exec("insert into example_tmq.t1 values(now,1)")
-    }
+            if err != nil {
+                panic(err)
+            }
+            time.Sleep(time.Millisecond * 100)
+        }
+    }()
+
     for i := 0; i < 5; i++ {
         ev := consumer.Poll(500)
         if ev != nil {
@@ -975,6 +979,7 @@ package main
 import (
     "database/sql"
     "fmt"
+    "time"
 
     "github.com/taosdata/driver-go/v3/common"
     tmqcommon "github.com/taosdata/driver-go/v3/common/tmq"
@@ -998,7 +1003,7 @@ func main() {
         "td.connect.pass":       "taosdata",
         "group.id":              "example",
         "client.id":             "example_consumer",
-        "auto.offset.reset":     "earliest",
+        "auto.offset.reset":     "latest",
     })
     if err != nil {
         panic(err)
@@ -1007,29 +1012,34 @@ func main() {
     if err != nil {
         panic(err)
     }
+
+    _, err = db.Exec("create table example_ws_tmq.t_all(ts timestamp," +
+        "c1 bool," +
+        "c2 tinyint," +
+        "c3 smallint," +
+        "c4 int," +
+        "c5 bigint," +
+        "c6 tinyint unsigned," +
+        "c7 smallint unsigned," +
+        "c8 int unsigned," +
+        "c9 bigint unsigned," +
+        "c10 float," +
+        "c11 double," +
+        "c12 binary(20)," +
+        "c13 nchar(20)" +
+        ")")
+    if err != nil {
+        panic(err)
+    }
     go func() {
-        _, err := db.Exec("create table example_ws_tmq.t_all(ts timestamp," +
+        for {
-            "c1 bool," +
+            _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')")
-            "c2 tinyint," +
+            if err != nil {
-            "c3 smallint," +
+                panic(err)
-            "c4 int," +
+            }
-            "c5 bigint," +
+            time.Sleep(time.Millisecond * 100)
-            "c6 tinyint unsigned," +
-            "c7 smallint unsigned," +
-            "c8 int unsigned," +
-            "c9 bigint unsigned," +
-            "c10 float," +
-            "c11 double," +
-            "c12 binary(20)," +
-            "c13 nchar(20)" +
-            ")")
-        if err != nil {
-            panic(err)
-        }
-        _, err = db.Exec("insert into example_ws_tmq.t_all values(now,true,2,3,4,5,6,7,8,9,10.123,11.123,'binary','nchar')")
-        if err != nil {
-            panic(err)
         }
+
     }()
     for i := 0; i < 5; i++ {
         ev := consumer.Poll(500)

docs/zh/08-connector/26-rust.mdx

@@ -447,7 +447,7 @@ consumer.unsubscribe().await;
 
 - `group.id`: 同一个消费者组，将以至少消费一次的方式进行消息负载均衡。
 - `client.id`: 可选的订阅客户端识别项。
-- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认为从头订阅。注意，此选项在同一个 `group.id` 中仅生效一次。
+- `auto.offset.reset`: 可选初始化订阅起点， *earliest* 为从头开始订阅， *latest* 为仅从最新数据开始订阅，默认值根据 TDengine 版本有所不同，详细参见 [数据订阅](https://docs.taosdata.com/develop/tmq/)。注意，此选项在同一个 `group.id` 中仅生效一次。
 - `enable.auto.commit`: 当设置为 `true` 时，将启用自动标记模式，当对数据一致性不敏感时，可以启用此方式。
 - `auto.commit.interval.ms`: 自动标记的时间间隔。
 

docs/zh/14-reference/12-config/index.md

@@ -648,7 +648,16 @@ charset 的有效值是 UTF-8。
 | 适用范围 | 仅客户端适用                    |
 | 含义     | schemaless 自定义的子表名的 key |
 | 类型     | 字符串                          |
-| 缺省值   | 无                              |
+| 缺省值   | 无  
+
+### smlAutoChildTableNameDelimiter
+
+| 属性     | 说明                            |
+| -------- | ------------------------------- |
+| 适用范围 | 仅客户端适用                    |
+| 含义     | schemaless tag之间的连接符，连起来作为子表名 |
+| 类型     | 字符串                          |
+| 缺省值   | 无  |
 
 ### smlTagName
 

docs/zh/14-reference/13-schemaless/13-schemaless.md

@@ -94,8 +94,11 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000
 :::tip
 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。
 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。
-:::tip 
+:::tip
-为了让用户可以指定生成的表名，可以通过在taos.cfg里配置 smlChildTableName 参数来指定。
+如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高：
+通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定。
+举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。
+通过在taos.cfg里配置 smlChildTableName 参数来指定。
 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。
 
 2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。

include/common/tcommon.h

@@ -55,8 +55,8 @@ typedef struct SSessionKey {
 } SSessionKey;
 
 typedef struct SVersionRange {
-  uint64_t minVer;
+  int64_t minVer;
-  uint64_t maxVer;
+  int64_t maxVer;
 } SVersionRange;
 
 static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) {

include/common/tglobal.h

@@ -179,6 +179,7 @@ extern char tsUdfdLdLibPath[];
 
 // schemaless
 extern char tsSmlChildTableName[];
+extern char tsSmlAutoChildTableNameDelimiter[];
 extern char tsSmlTagName[];
 extern bool tsSmlDot2Underline;
 extern char tsSmlTsDefaultName[];

include/common/tgrant.h

@@ -52,9 +52,9 @@ typedef enum {
 
 int32_t grantCheck(EGrantType grant);
 #ifndef TD_GRANT_OPTIMIZE
-int32_t grantAlterActiveCode(const char* old, const char* new, char* out, int8_t type);
+int32_t grantAlterActiveCode(const char* old, const char* newer, char* out, int8_t type);
 #else
-int32_t grantAlterActiveCode(int32_t did, const char* old, const char* new, char* out, int8_t type);
+int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
 #endif
 
 #ifndef GRANTS_CFG
@@ -114,4 +114,4 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* new, char
 }
 #endif
 
-#endif /*_TD_COMMON_GRANT_H_*/
+#endif /*_TD_COMMON_GRANT_H_*/

include/common/tmsg.h

@@ -2365,17 +2365,6 @@ int32_t tSerializeSCMCreateStreamReq(void* buf, int32_t bufLen, const SCMCreateS
 int32_t tDeserializeSCMCreateStreamReq(void* buf, int32_t bufLen, SCMCreateStreamReq* pReq);
 void    tFreeSCMCreateStreamReq(SCMCreateStreamReq* pReq);
 
-typedef struct {
-  char    name[TSDB_STREAM_FNAME_LEN];
-  int64_t streamId;
-  char*   sql;
-  char*   executorMsg;
-} SMVCreateStreamReq, SMSCreateStreamReq;
-
-typedef struct {
-  int64_t streamId;
-} SMVCreateStreamRsp, SMSCreateStreamRsp;
-
 enum {
   TOPIC_SUB_TYPE__DB = 1,
   TOPIC_SUB_TYPE__TABLE,
@@ -2397,16 +2386,9 @@ int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTo
 int32_t tDeserializeSCMCreateTopicReq(void* buf, int32_t bufLen, SCMCreateTopicReq* pReq);
 void    tFreeSCMCreateTopicReq(SCMCreateTopicReq* pReq);
 
-typedef struct {
-  int64_t topicId;
-} SCMCreateTopicRsp;
-
-int32_t tSerializeSCMCreateTopicRsp(void* buf, int32_t bufLen, const SCMCreateTopicRsp* pRsp);
-int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicRsp* pRsp);
-
 typedef struct {
   int64_t consumerId;
-} SMqConsumerLostMsg, SMqConsumerRecoverMsg, SMqConsumerClearMsg;
+} SMqConsumerRecoverMsg, SMqConsumerClearMsg;
 
 typedef struct {
   int64_t consumerId;
@@ -2418,6 +2400,7 @@ typedef struct {
   int8_t  autoCommit;
   int32_t autoCommitInterval;
   int8_t  resetOffsetCfg;
+  int8_t  enableReplay;
 } SCMSubscribeReq;
 
 static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@@ -2437,6 +2420,7 @@ static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubsc
   tlen += taosEncodeFixedI8(buf, pReq->autoCommit);
   tlen += taosEncodeFixedI32(buf, pReq->autoCommitInterval);
   tlen += taosEncodeFixedI8(buf, pReq->resetOffsetCfg);
+  tlen += taosEncodeFixedI8(buf, pReq->enableReplay);
 
   return tlen;
 }
@@ -2460,71 +2444,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq
   buf = taosDecodeFixedI8(buf, &pReq->autoCommit);
   buf = taosDecodeFixedI32(buf, &pReq->autoCommitInterval);
   buf = taosDecodeFixedI8(buf, &pReq->resetOffsetCfg);
-  return buf;
+  buf = taosDecodeFixedI8(buf, &pReq->enableReplay);
-}
-
-typedef struct SMqSubTopic {
-  int32_t vgId;
-  int64_t topicId;
-  SEpSet  epSet;
-} SMqSubTopic;
-
-typedef struct {
-  int32_t     topicNum;
-  SMqSubTopic topics[];
-} SCMSubscribeRsp;
-
-static FORCE_INLINE int32_t tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI32(buf, pRsp->topicNum);
-  for (int32_t i = 0; i < pRsp->topicNum; i++) {
-    tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId);
-    tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId);
-    tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet);
-  }
-  return tlen;
-}
-
-static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) {
-  buf = taosDecodeFixedI32(buf, &pRsp->topicNum);
-  for (int32_t i = 0; i < pRsp->topicNum; i++) {
-    buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId);
-    buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId);
-    buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet);
-  }
-  return buf;
-}
-
-typedef struct {
-  int64_t topicId;
-  int64_t consumerId;
-  int64_t consumerGroupId;
-  int64_t offset;
-  char*   sql;
-  char*   logicalPlan;
-  char*   physicalPlan;
-} SMVSubscribeReq;
-
-static FORCE_INLINE int32_t tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) {
-  int32_t tlen = 0;
-  tlen += taosEncodeFixedI64(buf, pReq->topicId);
-  tlen += taosEncodeFixedI64(buf, pReq->consumerId);
-  tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId);
-  tlen += taosEncodeFixedI64(buf, pReq->offset);
-  tlen += taosEncodeString(buf, pReq->sql);
-  tlen += taosEncodeString(buf, pReq->logicalPlan);
-  tlen += taosEncodeString(buf, pReq->physicalPlan);
-  return tlen;
-}
-
-static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) {
-  buf = taosDecodeFixedI64(buf, &pReq->topicId);
-  buf = taosDecodeFixedI64(buf, &pReq->consumerId);
-  buf = taosDecodeFixedI64(buf, &pReq->consumerGroupId);
-  buf = taosDecodeFixedI64(buf, &pReq->offset);
-  buf = taosDecodeString(buf, &pReq->sql);
-  buf = taosDecodeString(buf, &pReq->logicalPlan);
-  buf = taosDecodeString(buf, &pReq->physicalPlan);
   return buf;
 }
 
@@ -3620,6 +3540,7 @@ typedef struct {
   int64_t      consumerId;
   int64_t      timeout;
   STqOffsetVal reqOffset;
+  int8_t       enableReplay;
 } SMqPollReq;
 
 int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq);
@@ -3679,6 +3600,7 @@ typedef struct {
   SArray*      blockData;
   SArray*      blockTbName;
   SArray*      blockSchema;
+  int64_t      sleepTime;
 } SMqDataRsp;
 
 int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp);

include/common/tmsgdef.h

@@ -299,8 +299,8 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT, "sync-pre-snapshot", NULL, NULL)             // no longer used
+  TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT, "sync-prep-snapshot", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_SYNC_PRE_SNAPSHOT_REPLY, "sync-pre-snapshot-reply", NULL, NULL) // no longer used
+  TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
 

include/libs/executor/storageapi.h

@@ -227,6 +227,7 @@ typedef struct SStoreTqReader {
   bool (*tqReaderNextBlockInWal)();
   bool (*tqNextBlockImpl)();  // todo remove it
   SSDataBlock* (*tqGetResultBlock)();
+  int64_t (*tqGetResultBlockTime)();
 
   void (*tqReaderSetColIdList)();
   int32_t (*tqReaderSetQueryTableList)();

include/libs/nodes/querynodes.h

@@ -35,6 +35,7 @@ typedef struct SRawExprNode {
   char*     p;
   uint32_t  n;
   SNode*    pNode;
+  bool isPseudoColumn;
 } SRawExprNode;
 
 typedef struct SDataType {

include/libs/stream/tstreamFileState.h

@@ -79,6 +79,9 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v
 int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen);
 int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos);
 
+SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen);
+int32_t      recoverSesssion(SStreamFileState* pFileState, int64_t ckId);
+
 void sessionWinStateClear(SStreamFileState* pFileState);
 void sessionWinStateCleanup(void* pBuff);
 

include/libs/sync/sync.h

@@ -36,8 +36,7 @@ extern "C" {
 #define SYNC_DEL_WAL_MS              (1000 * 60)
 #define SYNC_ADD_QUORUM_COUNT        3
 #define SYNC_VNODE_LOG_RETENTION     (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
-#define SNAPSHOT_MAX_CLOCK_SKEW_MS   1000 * 10
+#define SNAPSHOT_WAIT_MS             1000 * 5
-#define SNAPSHOT_WAIT_MS             1000 * 30
 
 #define SYNC_MAX_RETRY_BACKOFF         5
 #define SYNC_LOG_REPL_RETRY_WAIT_MS    100
@@ -87,6 +86,11 @@ typedef enum {
   TAOS_SYNC_ROLE_ERROR = 2,
 } ESyncRole;
 
+typedef enum {
+  SYNC_FSM_STATE_COMPLETE = 0,
+  SYNC_FSM_STATE_INCOMPLETE,
+} ESyncFsmState;
+
 typedef struct SNodeInfo {
   int64_t   clusterId;
   int32_t   nodeId;
@@ -95,6 +99,12 @@ typedef struct SNodeInfo {
   ESyncRole nodeRole;
 } SNodeInfo;
 
+typedef struct SSyncTLV {
+  int32_t typ;
+  int32_t len;
+  char    val[];
+} SSyncTLV;
+
 typedef struct SSyncCfg {
   int32_t   totalReplicaNum;
   int32_t   replicaNum;
@@ -139,10 +149,13 @@ typedef struct SReConfigCbMeta {
 typedef struct SSnapshotParam {
   SyncIndex start;
   SyncIndex end;
+  SSyncTLV* data;
 } SSnapshotParam;
 
 typedef struct SSnapshot {
-  void*     data;
+  int32_t       type;
+  SSyncTLV* data;
+  ESyncFsmState state;
   SyncIndex lastApplyIndex;
   SyncTerm  lastApplyTerm;
   SyncIndex lastConfigIndex;
@@ -171,7 +184,7 @@ typedef struct SSyncFSM {
   void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm);
 
   int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader);
-  void (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+  int32_t (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
 
   int32_t (*FpSnapshotStartRead)(const struct SSyncFSM* pFsm, void* pReaderParam, void** ppReader);
   void (*FpSnapshotStopRead)(const struct SSyncFSM* pFsm, void* pReader);

include/libs/transport/trpc.h

@@ -163,6 +163,7 @@ int rpcReleaseHandle(void *handle, int8_t type);  // just release conn to rpc in
 // These functions will not be called in the child process
 int   rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
 int   rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
+int   rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int32_t timeoutMs);
 int   rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn);
 void *rpcAllocHandle();
 void  rpcSetIpWhite(void *thandl, void *arg);

include/util/taoserror.h

@@ -557,7 +557,7 @@ int32_t* taosGetErrno();
 // #define TSDB_CODE_SYN_TOO_MANY_FWDINFO       TAOS_DEF_ERROR_CODE(0, 0x0904) // 2.x
 // #define TSDB_CODE_SYN_MISMATCHED_PROTOCOL    TAOS_DEF_ERROR_CODE(0, 0x0905) // 2.x
 // #define TSDB_CODE_SYN_MISMATCHED_CLUSTERID   TAOS_DEF_ERROR_CODE(0, 0x0906) // 2.x
-// #define TSDB_CODE_SYN_MISMATCHED_SIGNATURE   TAOS_DEF_ERROR_CODE(0, 0x0907) // 2.x
+#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE      TAOS_DEF_ERROR_CODE(0, 0x0907)
 // #define TSDB_CODE_SYN_INVALID_CHECKSUM       TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x
 // #define TSDB_CODE_SYN_INVALID_MSGLEN         TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x
 // #define TSDB_CODE_SYN_INVALID_MSGTYPE        TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x
@@ -799,6 +799,8 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_TMQ_NEED_INITIALIZED           TAOS_DEF_ERROR_CODE(0, 0x4010)
 #define TSDB_CODE_TMQ_NO_COMMITTED               TAOS_DEF_ERROR_CODE(0, 0x4011)
 #define TSDB_CODE_TMQ_SAME_COMMITTED_VALUE       TAOS_DEF_ERROR_CODE(0, 0x4012)
+#define TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP     TAOS_DEF_ERROR_CODE(0, 0x4013)
+#define TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT         TAOS_DEF_ERROR_CODE(0, 0x4014)
 
 // stream
 #define TSDB_CODE_STREAM_TASK_NOT_EXIST          TAOS_DEF_ERROR_CODE(0, 0x4100)

include/util/tbloomfilter.h

@@ -24,6 +24,9 @@
 extern "C" {
 #endif
 
+#define HASH_FUNCTION_1 taosFastHash
+#define HASH_FUNCTION_2 taosDJB2Hash
+
 typedef struct SBloomFilter {
   uint32_t   hashFunctions;
   uint64_t   expectedEntries;
@@ -37,8 +40,9 @@ typedef struct SBloomFilter {
 } SBloomFilter;
 
 SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
+int32_t       tBloomFilterPutHash(SBloomFilter *pBF, uint64_t hash1, uint64_t hash2);
 int32_t       tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
-int32_t       tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf, uint32_t len);
+int32_t       tBloomFilterNoContain(const SBloomFilter *pBF, uint64_t h1, uint64_t h2);
 void          tBloomFilterDestroy(SBloomFilter *pBF);
 void          tBloomFilterDump(const SBloomFilter *pBF);
 bool          tBloomFilterIsFull(const SBloomFilter *pBF);

include/util/tdef.h

@@ -249,7 +249,7 @@ typedef enum ELogicConditionType {
 #define TSDB_PASSWORD_LEN      32
 #define TSDB_USET_PASSWORD_LEN 129
 #define TSDB_VERSION_LEN       32
-#define TSDB_LABEL_LEN         8
+#define TSDB_LABEL_LEN         12 
 #define TSDB_JOB_STATUS_LEN    32
 
 #define TSDB_CLUSTER_ID_LEN       40

include/util/tscalablebf.h

@@ -26,9 +26,12 @@ typedef struct SScalableBf {
   SArray  *bfArray;  // array of bloom filters
   uint32_t growth;
   uint64_t numBits;
+  _hash_fn_t hashFn1;
+  _hash_fn_t hashFn2;
 } SScalableBf;
 
 SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
+int32_t      tScalableBfPutNoCheck(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
 int32_t      tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
 int32_t      tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf, uint32_t len);
 void         tScalableBfDestroy(SScalableBf *pSBf);

source/client/src/clientSml.c

@@ -193,20 +193,46 @@ cleanup:
 }
 
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
-  size_t childTableNameLen = strlen(tsSmlChildTableName);
+  bool autoChildName = false;
-  if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS;
+  size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
-
+  if(delimiter > 0){
-  for (int i = 0; i < taosArrayGetSize(tags); i++) {
+    size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1);
-    SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
+    for (int i = 0; i < taosArrayGetSize(tags); i++) {
-    // handle child table name
+      SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
-    if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
+      totalNameLen += tag->length;
-      memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
+    }
-      strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN));
+    if(totalNameLen < TSDB_TABLE_NAME_LEN){
-      if(tsSmlDot2Underline){
+      autoChildName = true;
-        smlStrReplace(childTableName, strlen(childTableName));
+    }
+  }
+  if(autoChildName){
+    memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
+    for (int i = 0; i < taosArrayGetSize(tags); i++) {
+      SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
+      strncat(childTableName, tag->value, tag->length);
+      if(i != taosArrayGetSize(tags) - 1){
+        strcat(childTableName, tsSmlAutoChildTableNameDelimiter);
+      }
+    }
+    if(tsSmlDot2Underline){
+      smlStrReplace(childTableName, strlen(childTableName));
+    }
+  }else{
+    size_t childTableNameLen = strlen(tsSmlChildTableName);
+    if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS;
+
+    for (int i = 0; i < taosArrayGetSize(tags); i++) {
+      SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
+      // handle child table name
+      if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
+        memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
+        strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN));
+        if(tsSmlDot2Underline){
+          smlStrReplace(childTableName, strlen(childTableName));
+        }
+        taosArrayRemove(tags, i);
+        break;
       }
-      taosArrayRemove(tags, i);
-      break;
     }
   }
 

source/client/src/clientSmlJson.c

@@ -853,7 +853,6 @@ static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPr
   size_t  typeLen = strlen(type->valuestring);
   if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) {
     // seconds
-//    int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS;
     if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) {
       return tsInt64 * smlFactorS[toPrecision];
     }

source/client/src/clientTmq.c

@@ -62,7 +62,7 @@ struct tmq_conf_t {
   int8_t         resetOffset;
   int8_t         withTbName;
   int8_t         snapEnable;
-//  int32_t        snapBatchSize;
+  int8_t         replayEnable;
   uint16_t       port;
   int32_t        autoCommitInterval;
   char*          ip;
@@ -81,6 +81,7 @@ struct tmq_t {
   int8_t         autoCommit;
   int32_t        autoCommitInterval;
   int8_t         resetOffsetCfg;
+  int8_t         replayEnable;
   uint64_t       consumerId;
   tmq_commit_cb* commitCb;
   void*          commitCbUserParam;
@@ -89,19 +90,13 @@ struct tmq_t {
   SRWLatch        lock;
   int8_t  status;
   int32_t epoch;
-#if 0
-  int8_t  epStatus;
-  int32_t epSkipCnt;
-#endif
   // poll info
   int64_t pollCnt;
   int64_t totalRows;
-//  bool    needReportOffsetRows;
 
   // timer
   tmr_h       hbLiveTimer;
   tmr_h       epTimer;
-  tmr_h       reportTimer;
   tmr_h       commitTimer;
   STscObj*    pTscObj;       // connection
   SArray*     clientTopics;  // SArray<SMqClientTopic>
@@ -149,6 +144,8 @@ typedef struct {
   int32_t       vgStatus;
   int32_t       vgSkipCnt;              // here used to mark the slow vgroups
   int64_t       emptyBlockReceiveTs;    // once empty block is received, idle for ignoreCnt then start to poll data
+  int64_t       blockReceiveTs;         // once empty block is received, idle for ignoreCnt then start to poll data
+  int64_t       blockSleepForReplay;    // once empty block is received, idle for ignoreCnt then start to poll data
   bool          seekUpdated;            // offset is updated by seek operator, therefore, not update by vnode rsp.
   SEpSet        epSet;
 } SMqClientVg;
@@ -356,24 +353,6 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
 
-//  if (strcasecmp(key, "experimental.snapshot.batch.size") == 0) {
-//    conf->snapBatchSize = taosStr2int64(value);
-//    return TMQ_CONF_OK;
-//  }
-
-//  if (strcasecmp(key, "enable.heartbeat.background") == 0) {
-    //    if (strcasecmp(value, "true") == 0) {
-    //      conf->hbBgEnable = true;
-    //      return TMQ_CONF_OK;
-    //    } else if (strcasecmp(value, "false") == 0) {
-    //      conf->hbBgEnable = false;
-    //      return TMQ_CONF_OK;
-    //    } else {
-//    tscError("the default value of enable.heartbeat.background is true, can not be seted");
-//    return TMQ_CONF_INVALID;
-    //    }
-//  }
-
   if (strcasecmp(key, "td.connect.ip") == 0) {
     conf->ip = taosStrdup(value);
     return TMQ_CONF_OK;
@@ -394,6 +373,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     return TMQ_CONF_OK;
   }
 
+  if (strcasecmp(key, "enable.replay") == 0) {
+    if (strcasecmp(value, "true") == 0) {
+      conf->replayEnable = true;
+      return TMQ_CONF_OK;
+    } else if (strcasecmp(value, "false") == 0) {
+      conf->replayEnable = false;
+      return TMQ_CONF_OK;
+    } else {
+      return TMQ_CONF_INVALID;
+    }
+  }
+
   if (strcasecmp(key, "td.connect.db") == 0) {
     return TMQ_CONF_OK;
   }
@@ -729,6 +720,17 @@ void tmqAssignAskEpTask(void* param, void* tmrId) {
   taosMemoryFree(param);
 }
 
+void tmqReplayTask(void* param, void* tmrId) {
+  int64_t refId = *(int64_t*)param;
+  tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
+  if(tmq == NULL) goto END;
+
+  tsem_post(&tmq->rspSem);
+  taosReleaseRef(tmqMgmt.rsetId, refId);
+END:
+  taosMemoryFree(param);
+}
+
 void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
   int64_t refId = *(int64_t*)param;
   generateTimedTask(refId, TMQ_DELAYED_TASK__COMMIT);
@@ -1071,6 +1073,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->commitCb = conf->commitCb;
   pTmq->commitCbUserParam = conf->commitCbUserParam;
   pTmq->resetOffsetCfg = conf->resetOffset;
+  pTmq->replayEnable = conf->replayEnable;
+  if(conf->replayEnable){
+    pTmq->autoCommit = false;
+  }
   taosInitRWLatch(&pTmq->lock);
 
   // assign consumerId
@@ -1140,6 +1146,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
   req.autoCommit = tmq->autoCommit;
   req.autoCommitInterval = tmq->autoCommitInterval;
   req.resetOffsetCfg = tmq->resetOffsetCfg;
+  req.enableReplay = tmq->replayEnable;
 
   for (int32_t i = 0; i < sz; i++) {
     char* topic = taosArrayGetP(container, i);
@@ -1415,6 +1422,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic
         .vgStatus = pInfo ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE,
         .vgSkipCnt = 0,
         .emptyBlockReceiveTs = 0,
+        .blockReceiveTs = 0,
+        .blockSleepForReplay = 0,
         .numOfRows = pInfo ? pInfo->numOfRows : 0,
     };
 
@@ -1526,6 +1535,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl
   pReq->head.vgId = pVg->vgId;
   pReq->useSnapshot = tmq->useSnapshot;
   pReq->reqId = generateRequestId();
+  pReq->enableReplay = tmq->replayEnable;
 }
 
 SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
@@ -1686,6 +1696,12 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
         continue;
       }
 
+      if (tmq->replayEnable && taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) {  // less than 10ms
+        tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId,
+                 tmq->epoch, pVg->vgId, pVg->blockSleepForReplay);
+        continue;
+      }
+
       int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT);
       if (vgStatus == TMQ_VG_STATUS__WAIT) {
         int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1);
@@ -1807,6 +1823,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) {
           SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows);
           tmq->totalRows += numOfRows;
           pVg->emptyBlockReceiveTs = 0;
+          if(tmq->replayEnable){
+            pVg->blockReceiveTs = taosGetTimestampMs();
+            pVg->blockSleepForReplay = pRsp->rsp.sleepTime;
+            if(pVg->blockSleepForReplay > 0){
+              int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t));
+              *pRefId1 = tmq->refId;
+              taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, pRefId1, tmqMgmt.timer);
+            }
+          }
           tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64
                    ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64,
                    tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows,

source/common/src/tglobal.c

@@ -118,7 +118,8 @@ bool tsSmlDot2Underline = true;
 char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts";
 char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null";
 char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = "";  // user defined child table name can be specified in tag value.
-                                                     // If set to empty system will generate table name using MD5 hash.
+char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = "";
+                                                                   // If set to empty system will generate table name using MD5 hash.
 // true means that the name and order of cols in each line are the same(only for influx protocol)
 // bool    tsSmlDataFormat = false;
 // int32_t tsSmlBatchSize = 10000;
@@ -439,7 +440,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1;
-  if (cfgAddString(pCfg, "smlChildTableName", "", CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1;
+  if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1;
   if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1;
@@ -931,6 +933,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
     return -1;
   }
 
+  tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN);
   tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
   tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
   tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN);
@@ -1396,6 +1399,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) {
         cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype);
       } else if (strcasecmp("smlChildTableName", name) == 0) {
         tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN);
+      } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) {
+        tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN);
       } else if (strcasecmp("smlTagName", name) == 0) {
         tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN);
         //      } else if (strcasecmp("smlDataFormat", name) == 0) {

source/common/src/tmsg.c

@@ -4552,31 +4552,6 @@ void tFreeSCMCreateTopicReq(SCMCreateTopicReq *pReq) {
   }
 }
 
-int32_t tSerializeSCMCreateTopicRsp(void *buf, int32_t bufLen, const SCMCreateTopicRsp *pRsp) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
-
-  if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI64(&encoder, pRsp->topicId) < 0) return -1;
-  tEndEncode(&encoder);
-
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-}
-
-int32_t tDeserializeSCMCreateTopicRsp(void *buf, int32_t bufLen, SCMCreateTopicRsp *pRsp) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-
-  if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI64(&decoder, &pRsp->topicId) < 0) return -1;
-  tEndDecode(&decoder);
-
-  tDecoderClear(&decoder);
-  return 0;
-}
-
 int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -6219,6 +6194,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
   if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1;
   if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->enableReplay) < 0) return -1;
 
   tEndEncode(&encoder);
 
@@ -6255,6 +6231,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) {
   if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1;
   if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1;
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI8(&decoder, &pReq->enableReplay) < 0) return -1;
+  }
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);
@@ -8057,6 +8037,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
       }
     }
   }
+  if (tEncodeI64(pEncoder, pRsp->sleepTime) < 0) return -1;
   return 0;
 }
 
@@ -8102,6 +8083,8 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
       }
     }
   }
+  if (tDecodeI64(pDecoder, &pRsp->sleepTime) < 0) return -1;
+
   return 0;
 }
 

source/common/src/ttime.c

@@ -665,6 +665,9 @@ int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* durati
   if (*unit == 'n' || *unit == 'y') {
     return 0;
   }
+  if(isdigit(*unit)) {
+    *unit = getPrecisionUnit(timePrecision);
+  }
 
   return getDuration(*duration, *unit, duration, timePrecision);
 }

source/dnode/mgmt/mgmt_dnode/src/dmHandle.c

@@ -160,7 +160,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
 
   SEpSet epSet = {0};
   dmGetMnodeEpSet(pMgmt->pData, &epSet);
-  rpcSendRecv(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp);
+  rpcSendRecvWithTimeout(pMgmt->msgCb.clientRpc, &epSet, &rpcMsg, &rpcRsp, 5000);
   if (rpcRsp.code != 0) {
     dmRotateMnodeEpSet(pMgmt->pData);
     char tbuf[256];

source/dnode/mgmt/mgmt_mnode/src/mmHandle.c

@@ -238,7 +238,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
 
@@ -246,7 +246,7 @@ SArray *mmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT_REPLY, mmPutMsgToSyncRdQueue, 1) == NULL) goto _OVER;
 
   code = 0;
 

source/dnode/mgmt/mgmt_mnode/src/mmInt.c

@@ -32,8 +32,13 @@ static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) {
 
   if (!option.deploy) {
     *required = mmDeployRequired(pInput);
+    if (*required) {
+      dInfo("deploy mnode required. dnodeId:%d<=0, clusterId:%" PRId64 "<=0, localEp:%s==firstEp",
+            pInput->pData->dnodeId, pInput->pData->clusterId, tsLocalEp);
+    }
   } else {
     *required = true;
+    dInfo("deploy mnode required. option deploy:%d", option.deploy);
   }
 
   return 0;

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -56,6 +56,7 @@ typedef struct {
   int32_t       vgVersion;
   int32_t       refCount;
   int8_t        dropped;
+  int8_t        failed;
   int8_t        disable;
   int32_t       diskPrimary;
   int32_t       toVgId;

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -30,9 +30,11 @@ void vmGetVnodeLoads(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo, bool isReset) {
     if (ppVnode == NULL || *ppVnode == NULL) continue;
 
     SVnodeObj *pVnode = *ppVnode;
-    SVnodeLoad vload = {0};
+    SVnodeLoad vload = {.vgId = pVnode->vgId};
-    vnodeGetLoad(pVnode->pImpl, &vload);
+    if (!pVnode->failed) {
-    if (isReset) vnodeResetLoad(pVnode->pImpl, &vload);
+      vnodeGetLoad(pVnode->pImpl, &vload);
+      if (isReset) vnodeResetLoad(pVnode->pImpl, &vload);
+    }
     taosArrayPush(pInfo->pVloads, &vload);
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }
@@ -52,9 +54,11 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
     if (ppVnode == NULL || *ppVnode == NULL) continue;
 
     SVnodeObj     *pVnode = *ppVnode;
-    SVnodeLoadLite vload = {0};
+    if (!pVnode->failed) {
-    if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
+      SVnodeLoadLite vload = {0};
-      taosArrayPush(pInfo->pVloads, &vload);
+      if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
+        taosArrayPush(pInfo->pVloads, &vload);
+      }
     }
     pIter = taosHashIterate(pMgmt->hash, pIter);
   }
@@ -278,7 +282,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   vmGenerateWrapperCfg(pMgmt, &req, &wrapperCfg);
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
-  if (pVnode != NULL) {
+  if (pVnode != NULL && !pVnode->failed) {
     dError("vgId:%d, already exist", req.vgId);
     tFreeSCreateVnodeReq(&req);
     vmReleaseVnode(pMgmt, pVnode);
@@ -287,7 +291,9 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     return 0;
   }
 
-  wrapperCfg.diskPrimary = vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId);
+  ASSERT(pVnode == NULL || pVnode->failed);
+
+  wrapperCfg.diskPrimary = pVnode ? pVnode->diskPrimary : vmAllocPrimaryDisk(pMgmt, vnodeCfg.vgId);
   int32_t diskPrimary = wrapperCfg.diskPrimary;
 
   snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId);
@@ -299,7 +305,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     goto _OVER;
   }
 
-  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb);
+  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, true);
   if (pImpl == NULL) {
     dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
     code = terrno;
@@ -364,9 +370,10 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
         TMSG_INFO(pMsg->msgType));
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
     dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
+    if (pVnode) vmReleaseVnode(pMgmt, pVnode);
     return -1;
   }
 
@@ -445,7 +452,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   dInfo("vgId:%d, begin to open vnode", vgId);
-  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb);
+  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false);
   if (pImpl == NULL) {
     dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr());
     return -1;
@@ -481,9 +488,10 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
           req.vgId, TMSG_INFO(pMsg->msgType));
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
     dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
+    if (pVnode) vmReleaseVnode(pMgmt, pVnode);
     return -1;
   }
 
@@ -523,9 +531,10 @@ int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   dInfo("vgId:%d, vnode write disable:%d", req.vgId, req.disable);
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
     dError("vgId:%d, failed to disable write since %s", req.vgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
+    if (pVnode) vmReleaseVnode(pMgmt, pVnode);
     return -1;
   }
 
@@ -555,9 +564,10 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd,
         req.dstVgId);
   pVnode = vmAcquireVnode(pMgmt, srcVgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
     dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
+    if (pVnode) vmReleaseVnode(pMgmt, pVnode);
     return -1;
   }
 
@@ -592,7 +602,7 @@ int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   dInfo("vgId:%d, open vnode", dstVgId);
-  SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb);
+  SVnode *pImpl = vnodeOpen(dstPath, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false);
 
   if (pImpl == NULL) {
     dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr());
@@ -669,9 +679,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
     dError("vgId:%d, failed to alter replica since %s", vgId, terrstr());
     terrno = TSDB_CODE_VND_NOT_EXIST;
+    if (pVnode) vmReleaseVnode(pMgmt, pVnode);
     return -1;
   }
 
@@ -696,7 +707,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
   }
 
   dInfo("vgId:%d, begin to open vnode", vgId);
-  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb);
+  SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false);
   if (pImpl == NULL) {
     dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr());
     return -1;
@@ -848,14 +859,14 @@ SArray *vmGetMsgHandles() {
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_BATCH, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_FORCE_FOLLOWER, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER;
 
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_HEARTBEAT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
   if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
-  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PRE_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
+  if (dmSetMgmtHandle(pArray, TDMT_SYNC_PREP_SNAPSHOT_REPLY, vmPutMsgToSyncRdQueue, 0) == NULL) goto _OVER;
 
   code = 0;
 

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -112,6 +112,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
   pVnode->diskPrimary = pCfg->diskPrimary;
   pVnode->refCount = 0;
   pVnode->dropped = 0;
+  pVnode->failed = 0;
   pVnode->path = taosStrdup(pCfg->path);
   pVnode->pImpl = pImpl;
 
@@ -121,11 +122,15 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
     return -1;
   }
 
-  if (vmAllocQueue(pMgmt, pVnode) != 0) {
+  if (pImpl) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    if (vmAllocQueue(pMgmt, pVnode) != 0) {
-    taosMemoryFree(pVnode->path);
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
-    taosMemoryFree(pVnode);
+      taosMemoryFree(pVnode->path);
-    return -1;
+      taosMemoryFree(pVnode);
+      return -1;
+    }
+  } else {
+    pVnode->failed = 1;
   }
 
   taosThreadRwlockWrlock(&pMgmt->lock);
@@ -267,12 +272,14 @@ static void *vmOpenVnodeInThread(void *param) {
     int32_t diskPrimary = pCfg->diskPrimary;
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
 
-    SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb);
+    SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, false);
 
     if (pImpl == NULL) {
       dError("vgId:%d, failed to open vnode by thread:%d since %s", pCfg->vgId, pThread->threadIndex, terrstr());
-      pThread->failed++;
+      if (terrno != TSDB_CODE_NEED_RETRY) {
-      continue;
+        pThread->failed++;
+        continue;
+      }
     }
 
     if (vmOpenVnode(pMgmt, pCfg, pImpl) != 0) {
@@ -379,6 +386,7 @@ static void *vmCloseVnodeInThread(void *param) {
 
   for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
     SVnodeObj *pVnode = pThread->ppVnodes[v];
+    if (pVnode->failed) continue;
 
     char stepDesc[TSDB_STEP_DESC_LEN] = {0};
     snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to close, %d of %d have been closed", pVnode->vgId,
@@ -473,7 +481,9 @@ static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) {
   if (ppVnodes != NULL) {
     for (int32_t i = 0; i < numOfVnodes; ++i) {
       SVnodeObj *pVnode = ppVnodes[i];
-      vnodeSyncCheckTimeout(pVnode->pImpl);
+      if (!pVnode->failed) {
+        vnodeSyncCheckTimeout(pVnode->pImpl);
+      }
       vmReleaseVnode(pMgmt, pVnode);
     }
     taosMemoryFree(ppVnodes);
@@ -605,6 +615,12 @@ static void *vmRestoreVnodeInThread(void *param) {
 
   for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
     SVnodeObj *pVnode = pThread->ppVnodes[v];
+    if (pVnode->failed) {
+      dError("vgId:%d, skip restoring vnode in failure mode.", pVnode->vgId);
+      continue;
+    }
+
+    ASSERT(pVnode->pImpl);
 
     char stepDesc[TSDB_STEP_DESC_LEN] = {0};
     snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been restored", pVnode->vgId,

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -187,9 +187,9 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
   pHead->vgId = ntohl(pHead->vgId);
 
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
-  if (pVnode == NULL) {
+  if (pVnode == NULL || pVnode->failed) {
-    dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
+    dGDebug("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
-           terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
+            terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
     terrno = (terrno != 0) ? terrno : -1;
     return terrno;
   }
@@ -316,7 +316,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
 int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
   int32_t    size = -1;
   SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
-  if (pVnode != NULL) {
+  if (pVnode != NULL && !pVnode->failed) {
     switch (qtype) {
       case WRITE_QUEUE:
         size = taosQueueItemSize(pVnode->pWriteW.queue);
@@ -339,8 +339,8 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
       default:
         break;
     }
-    vmReleaseVnode(pMgmt, pVnode);
   }
+  if (pVnode) vmReleaseVnode(pMgmt, pVnode);
   if (size < 0) {
     dTrace("vgId:%d, can't get size from queue since %s, qtype:%d", vgId, terrstr(), qtype);
     size = 0;

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -115,7 +115,9 @@ int32_t dmRunDnode(SDnode *pDnode);
 int32_t dmInitServer(SDnode *pDnode);
 void    dmCleanupServer(SDnode *pDnode);
 int32_t dmInitClient(SDnode *pDnode);
+int32_t dmInitStatusClient(SDnode *pDnode);
 void    dmCleanupClient(SDnode *pDnode);
+void    dmCleanupStatusClient(SDnode *pDnode);
 SMsgCb  dmGetMsgcb(SDnode *pDnode);
 int32_t dmInitMsgHandle(SDnode *pDnode);
 int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -20,8 +20,8 @@
 #include "qworker.h"
 #include "tstream.h"
 #ifdef TD_TSZ
-#include "tglobal.h"
 #include "tcompression.h"
+#include "tglobal.h"
 #endif
 
 int32_t dmInitDnode(SDnode *pDnode) {
@@ -66,7 +66,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
     goto _OVER;
   }
 
-  if(dmInitModule(pDnode) != 0) {
+  if (dmInitModule(pDnode) != 0) {
     goto _OVER;
   }
 
@@ -91,6 +91,7 @@ void dmCleanupDnode(SDnode *pDnode) {
   if (pDnode == NULL) return;
 
   dmCleanupClient(pDnode);
+  dmCleanupStatusClient(pDnode);
   dmCleanupServer(pDnode);
   dmClearVars(pDnode);
   rpcCleanup();

source/dnode/mgmt/node_mgmt/src/dmTransport.c

@@ -358,6 +358,50 @@ int32_t dmInitClient(SDnode *pDnode) {
   dDebug("dnode rpc client is initialized");
   return 0;
 }
+int32_t dmInitStatusClient(SDnode *pDnode) {
+  SDnodeTrans *pTrans = &pDnode->trans;
+
+  SRpcInit rpcInit = {0};
+  rpcInit.label = "DND-STATUS";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
+  rpcInit.sessions = 1024;
+  rpcInit.connType = TAOS_CONN_CLIENT;
+  rpcInit.user = TSDB_DEFAULT_USER;
+  rpcInit.idleTime = tsShellActivityTimer * 1000;
+  rpcInit.parent = pDnode;
+  rpcInit.rfp = rpcRfp;
+  rpcInit.compressSize = tsCompressMsgSize;
+
+  rpcInit.retryMinInterval = tsRedirectPeriod;
+  rpcInit.retryStepFactor = tsRedirectFactor;
+  rpcInit.retryMaxInterval = tsRedirectMaxPeriod;
+  rpcInit.retryMaxTimeout = tsMaxRetryWaitTime;
+
+  rpcInit.failFastInterval = 5000;  // interval threshold(ms)
+  rpcInit.failFastThreshold = 3;    // failed threshold
+  rpcInit.ffp = dmFailFastFp;
+
+  int32_t connLimitNum = 100;
+  connLimitNum = TMAX(connLimitNum, 10);
+  connLimitNum = TMIN(connLimitNum, 500);
+
+  rpcInit.connLimitNum = connLimitNum;
+  rpcInit.connLimitLock = 1;
+  rpcInit.supportBatch = 1;
+  rpcInit.batchSize = 8 * 1024;
+  rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
+  taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
+
+  // pTrans->statusClientRpc = rpcOpen(&rpcInit);
+  // if (pTrans->statusClientRpc == NULL) {
+  //   dError("failed to init dnode rpc status client");
+  //   return -1;
+  // }
+
+  dDebug("dnode rpc status client is initialized");
+  return 0;
+}
 
 void dmCleanupClient(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;
@@ -367,6 +411,14 @@ void dmCleanupClient(SDnode *pDnode) {
     dDebug("dnode rpc client is closed");
   }
 }
+void dmCleanupStatusClient(SDnode *pDnode) {
+  SDnodeTrans *pTrans = &pDnode->trans;
+  // if (pTrans->statusClientRpc) {
+  //   rpcClose(pTrans->statusClientRpc);
+  //   pTrans->statusClientRpc = NULL;
+  //   dDebug("dnode rpc status client is closed");
+  // }
+}
 
 int32_t dmInitServer(SDnode *pDnode) {
   SDnodeTrans *pTrans = &pDnode->trans;

source/dnode/mnode/impl/inc/mndDef.h

@@ -491,32 +491,6 @@ typedef struct {
   char           filterTb[TSDB_TABLE_NAME_LEN];
 } SShowObj;
 
-typedef struct {
-  int64_t id;
-  int8_t  type;
-  int8_t  replica;
-  int16_t numOfColumns;
-  int32_t rowSize;
-  int32_t numOfRows;
-  int32_t numOfReads;
-  int32_t payloadLen;
-  void*   pIter;
-  SMnode* pMnode;
-  char    db[TSDB_DB_FNAME_LEN];
-  int16_t offset[TSDB_MAX_COLUMNS];
-  int32_t bytes[TSDB_MAX_COLUMNS];
-  char    payload[];
-} SSysTableRetrieveObj;
-
-typedef struct {
-  char    key[TSDB_PARTITION_KEY_LEN];
-  int64_t dbUid;
-  int64_t offset;
-} SMqOffsetObj;
-
-int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset);
-void*   tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset);
-
 typedef struct {
   char           name[TSDB_TOPIC_FNAME_LEN];
   char           db[TSDB_DB_FNAME_LEN];

source/dnode/mnode/impl/src/mndConsumer.c

@@ -18,6 +18,7 @@
 #include "mndPrivilege.h"
 #include "mndVgroup.h"
 #include "mndShow.h"
+#include "mndDb.h"
 #include "mndSubscribe.h"
 #include "mndTopic.h"
 #include "mndTrans.h"
@@ -124,30 +125,55 @@ void mndRebCntDec() {
   }
 }
 
-static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser) {
+static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) {
-  int32_t numOfTopics = taosArrayGetSize(pTopicList);
+  SMqTopicObj *pTopic = NULL;
+  int32_t code = 0;
 
+  int32_t numOfTopics = taosArrayGetSize(pTopicList);
   for (int32_t i = 0; i < numOfTopics; i++) {
     char        *pOneTopic = taosArrayGetP(pTopicList, i);
-    SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pOneTopic);
+    pTopic = mndAcquireTopic(pMnode, pOneTopic);
     if (pTopic == NULL) {  // terrno has been set by callee function
-      return -1;
+      code = -1;
+      goto FAILED;
     }
 
     if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
-      mndReleaseTopic(pMnode, pTopic);
+      code = -1;
-      return -1;
+      goto FAILED;
+    }
+
+    if(enableReplay){
+      if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){
+        code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT;
+        goto FAILED;
+      }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) {
+        SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db);
+        if (pDb == NULL) {
+          code = -1;
+          goto FAILED;
+        }
+        if (pDb->cfg.numOfVgroups != 1) {
+          mndReleaseDb(pMnode, pDb);
+          code = TSDB_CODE_TMQ_REPLAY_NEED_ONE_VGROUP;
+          goto FAILED;
+        }
+        mndReleaseDb(pMnode, pDb);
+      }
     }
 
     mndTransSetDbName(pTrans, pOneTopic, NULL);
     if(mndTransCheckConflict(pMnode, pTrans) != 0){
-      mndReleaseTopic(pMnode, pTopic);
+      code = -1;
-      return -1;
+      goto FAILED;
     }
     mndReleaseTopic(pMnode, pTopic);
   }
 
   return 0;
+FAILED:
+  mndReleaseTopic(pMnode, pTopic);
+  return code;
 }
 
 static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
@@ -177,7 +203,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
   if (pTrans == NULL) {
     goto FAIL;
   }
-  if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user) != 0){
+  if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0){
     goto FAIL;
   }
 
@@ -697,7 +723,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
     goto _over;
   }
 
-  code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user);
+  code = validateTopics(pTrans, pTopicList, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
   if (code != TSDB_CODE_SUCCESS) {
     goto _over;
   }

source/dnode/mnode/impl/src/mndDump.c

@@ -330,24 +330,6 @@ void dumpSubscribe(SSdb *pSdb, SJson *json) {
   }
 }
 
-void dumpOffset(SSdb *pSdb, SJson *json) {
-  void  *pIter = NULL;
-  SJson *items = tjsonAddArrayToObject(json, "offsets");
-
-  while (1) {
-    SMqOffsetObj *pObj = NULL;
-    pIter = sdbFetch(pSdb, SDB_OFFSET, pIter, (void **)&pObj);
-    if (pIter == NULL) break;
-
-    SJson *item = tjsonCreateObject();
-    tjsonAddItemToArray(items, item);
-    tjsonAddStringToObject(item, "key", pObj->key);
-    tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
-    tjsonAddStringToObject(item, "offset", i642str(pObj->offset));
-    sdbRelease(pSdb, pObj);
-  }
-}
-
 void dumpStream(SSdb *pSdb, SJson *json) {
   void  *pIter = NULL;
   SJson *items = tjsonAddArrayToObject(json, "streams");
@@ -608,7 +590,7 @@ void mndDumpSdb() {
   dumpTopic(pSdb, json);
   dumpConsumer(pSdb, json);
   dumpSubscribe(pSdb, json);
-  dumpOffset(pSdb, json);
+//  dumpOffset(pSdb, json);
   dumpStream(pSdb, json);
   dumpAcct(pSdb, json);
   dumpAuth(pSdb, json);

source/dnode/mnode/impl/src/mndStb.c

@@ -858,22 +858,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   }
   return 0;
 }
-static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) {
+static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) {
-  char    randStr[TSDB_COL_NAME_LEN] = {0};
+  SName name = {0};
-  int32_t left = TSDB_COL_NAME_LEN - strlen(tagname) - 1;
+  tNameFromString(&name, stbname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  if (left <= 1) {
+  return snprintf(fullname, TSDB_INDEX_FNAME_LEN, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name));
-    sprintf(fullname, "%s.%s", dbname, tagname);
-  } else {
-    int8_t start = left < 8 ? 0 : 8;
-    int8_t end = left >= 24 ? 24 : left - 1;
-    // gen rand str len [base:end]
-    // note: ignore rand performance issues
-    int64_t len = taosRand() % (end - start + 1) + start;
-    taosRandStr2(randStr, len);
-    sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr);
-  }
-
-  return 0;
 }
 
 static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
@@ -889,7 +877,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
   if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
 
   SSchema *pSchema = &(stbObj.pTags[0]);
-  mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name);
+  mndGenIdxNameForFirstTag(fullIdxName, pDb->name, stbObj.name, pSchema->name);
   SSIdx idx = {0};
   if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
     terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;

source/dnode/mnode/impl/src/mndSync.c

@@ -286,9 +286,10 @@ int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pRe
   return 0;
 }
 
-static void mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+static int32_t mndSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) {
   SMnode *pMnode = pFsm->data;
   sdbGetCommitInfo(pMnode->pSdb, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, &pSnapshot->lastConfigIndex);
+  return 0;
 }
 
 void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) {

source/dnode/mnode/impl/src/mndTopic.c

@@ -298,11 +298,6 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic
   atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime);
   atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
 
-  /*taosWLockLatch(&pOldTopic->lock);*/
-
-  // TODO handle update
-
-  /*taosWUnLockLatch(&pOldTopic->lock);*/
   return 0;
 }
 
@@ -320,23 +315,6 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
   sdbRelease(pSdb, pTopic);
 }
 
-static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) {
-  int32_t contLen = sizeof(SDDropTopicReq);
-
-  SDDropTopicReq *pDrop = taosMemoryCalloc(1, contLen);
-  if (pDrop == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  pDrop->head.contLen = htonl(contLen);
-  pDrop->head.vgId = htonl(pVgroup->vgId);
-  memcpy(pDrop->name, pTopic->name, TSDB_TOPIC_FNAME_LEN);
-  pDrop->tuid = htobe64(pTopic->uid);
-
-  return pDrop;
-}
-
 static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
   terrno = TSDB_CODE_MND_INVALID_TOPIC;
 

source/dnode/vnode/inc/vnode.h

@@ -58,7 +58,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
 int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
                              int32_t diskPrimary, STfs *pTfs);
 void    vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs);
-SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb);
+SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force);
 void    vnodePreClose(SVnode *pVnode);
 void    vnodePostClose(SVnode *pVnode);
 void    vnodeSyncCheckTimeout(SVnode *pVnode);
@@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode);
 int32_t vnodeStart(SVnode *pVnode);
 void    vnodeStop(SVnode *pVnode);
 int64_t vnodeGetSyncHandle(SVnode *pVnode);
-void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
+int32_t   vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
 void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables);
 int32_t   vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
 int32_t   vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
@@ -226,6 +226,7 @@ typedef struct STqReader {
   int64_t         cachedSchemaUid;
   SSchemaWrapper *pSchemaWrapper;
   SSDataBlock    *pResBlock;
+  int64_t         lastTs;
 } STqReader;
 
 STqReader *tqReaderOpen(SVnode *pVnode);
@@ -244,6 +245,7 @@ bool         tqNextBlockInWal(STqReader *pReader, const char *idstr);
 bool         tqNextBlockImpl(STqReader *pReader, const char *idstr);
 SWalReader  *tqGetWalReader(STqReader *pReader);
 SSDataBlock *tqGetResultBlock(STqReader *pReader);
+int64_t      tqGetResultBlockTime(STqReader *pReader);
 
 int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id);
 int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver);
@@ -257,11 +259,11 @@ int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg);
 int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
 
 // SVSnapReader
-int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader);
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader);
 void    vnodeSnapReaderClose(SVSnapReader *pReader);
 int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData);
 // SVSnapWriter
-int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter);
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter);
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot);
 int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData);
 

source/dnode/vnode/src/inc/tq.h

@@ -46,6 +46,7 @@ typedef struct STqOffsetStore STqOffsetStore;
 // tqPush
 #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1)
 #define STREAM_EXEC_TASK_STATUS_CHECK_ID     (-2)
+#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
 
 // tqExec
 typedef struct {
@@ -90,6 +91,10 @@ typedef struct {
   STqExecHandle    execHandle;  // exec
   SRpcMsg*         msg;
   tq_handle_status status;
+
+  // for replay
+  SSDataBlock* block;
+  int64_t      blockTime;
 } STqHandle;
 
 struct STQ {
@@ -107,17 +112,13 @@ struct STQ {
   SStreamMeta*    pStreamMeta;
 };
 
-typedef struct {
-  int32_t size;
-} STqOffsetHead;
-
 int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle);
 int32_t tDecodeSTqHandle(SDecoder* pDecoder, STqHandle* pHandle);
 void    tqDestroyTqHandle(void* data);
 
 // tqRead
 int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* offset);
-int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset);
+int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest);
 int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId);
 
 // tqExec

source/dnode/vnode/src/inc/tsdb.h

@@ -263,6 +263,7 @@ int32_t tsdbFSRollback(STsdb *pTsdb);
 int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFS);
 int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS);
 void    tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS);
+void    tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t);
 
 int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet);
 int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile);
@@ -672,6 +673,42 @@ struct SDelFWriter {
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
 
+typedef struct STSnapRange STSnapRange;
+typedef TARRAY2(STSnapRange *) TSnapRangeArray;  // disjoint snap ranges
+
+// util
+int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
+int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
+void    tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap);
+SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges);
+
+// snap partition list
+typedef TARRAY2(SVersionRange) SVerRangeList;
+typedef struct STsdbSnapPartition STsdbSnapPartition;
+typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList;
+// util
+STsdbSnapPartList *tsdbSnapPartListCreate();
+void               tsdbSnapPartListDestroy(STsdbSnapPartList **ppList);
+int32_t            tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
+int32_t            tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
+int32_t            tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges);
+
+enum {
+  TSDB_SNAP_RANGE_TYP_HEAD = 0,
+  TSDB_SNAP_RANGE_TYP_DATA,
+  TSDB_SNAP_RANGE_TYP_SMA,
+  TSDB_SNAP_RANGE_TYP_TOMB,
+  TSDB_SNAP_RANGE_TYP_STT,
+  TSDB_SNAP_RANGE_TYP_MAX,
+};
+
+struct STsdbSnapPartition {
+  int64_t       fid;
+  int8_t        stat;
+  SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX];
+};
+
+// snap read
 struct STsdbReadSnap {
   SMemTable     *pMem;
   SQueryNode    *pNode;
@@ -989,6 +1026,15 @@ struct STsdbFilterInfo {
   TABLEID tbid;
 };
 
+typedef enum {
+  TSDB_FS_STATE_NORMAL = 0,
+  TSDB_FS_STATE_INCOMPLETE,
+} ETsdbFsState;
+
+// utils
+ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode);
+int32_t      tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -202,7 +202,7 @@ typedef struct SMetaInfo {
 int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader);
 
 // tsdb
-int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback);
+int     tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback, bool force);
 int     tsdbClose(STsdb** pTsdb);
 int32_t tsdbBegin(STsdb* pTsdb);
 // int32_t tsdbPrepareCommit(STsdb* pTsdb);
@@ -267,7 +267,7 @@ int32_t tqProcessTaskScanHistoryFinishRsp(STQ* pTq, SRpcMsg* pMsg);
 // sma
 int32_t smaInit();
 void    smaCleanUp();
-int32_t smaOpen(SVnode* pVnode, int8_t rollback);
+int32_t smaOpen(SVnode* pVnode, int8_t rollback, bool force);
 int32_t smaClose(SSma* pSma);
 int32_t smaBegin(SSma* pSma);
 int32_t smaPrepareAsyncCommit(SSma* pSma);
@@ -295,11 +295,12 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr
 int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
 int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback);
 // STsdbSnapReader ========================================
-int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** ppReader);
+int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges,
+                           STsdbSnapReader** ppReader);
 int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader);
 int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData);
 // STsdbSnapWriter ========================================
-int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter);
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRanges, STsdbSnapWriter** ppWriter);
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr);
 int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter);
 int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
@@ -356,8 +357,9 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead
 int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader);
 int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData);
 // SRSmaSnapWriter ========================================
-int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter);
+int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRanges, SRSmaSnapWriter** ppWriter);
 int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData);
+int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter);
 int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback);
 
 typedef struct {
@@ -497,6 +499,7 @@ struct SSma {
 #define SMA_RSMA_TSDB0(s) ((s)->pVnode->pTsdb)
 #define SMA_RSMA_TSDB1(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L0])
 #define SMA_RSMA_TSDB2(s) ((s)->pRSmaTsdb[TSDB_RETENTION_L1])
+#define SMA_RSMA_GET_TSDB(pVnode, level) ((level == 0) ? pVnode->pTsdb : pVnode->pSma->pRSmaTsdb[level - 1])
 
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);

source/dnode/vnode/src/sma/smaOpen.c

@@ -30,7 +30,7 @@ static int32_t rsmaRestore(SSma *pSma);
     pKeepCfg->keepTimeOffset = 0;                                                                 \
   } while (0)
 
-#define SMA_OPEN_RSMA_IMPL(v, l)                                                             \
+#define SMA_OPEN_RSMA_IMPL(v, l, force)                                                             \
   do {                                                                                       \
     SRetention *r = (SRetention *)VND_RETENTIONS(v) + l;                                     \
     if (!RETENTION_VALID(r)) {                                                               \
@@ -42,7 +42,7 @@ static int32_t rsmaRestore(SSma *pSma);
     }                                                                                        \
     code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l);                            \
     TSDB_CHECK_CODE(code, lino, _exit);                                                      \
-    if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback) < 0) { \
+    if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \
       code = terrno;                                                                         \
       TSDB_CHECK_CODE(code, lino, _exit);                                                    \
     }                                                                                        \
@@ -118,7 +118,7 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty
   return terrno;
 }
 
-int32_t smaOpen(SVnode *pVnode, int8_t rollback) {
+int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
   int32_t   code = 0;
   int32_t   lino = 0;
   STsdbCfg *pCfg = &pVnode->config.tsdbCfg;
@@ -139,11 +139,11 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback) {
     STsdbKeepCfg keepCfg = {0};
     for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
       if (i == TSDB_RETENTION_L0) {
-        SMA_OPEN_RSMA_IMPL(pVnode, 0);
+        SMA_OPEN_RSMA_IMPL(pVnode, 0, force);
       } else if (i == TSDB_RETENTION_L1) {
-        SMA_OPEN_RSMA_IMPL(pVnode, 1);
+        SMA_OPEN_RSMA_IMPL(pVnode, 1, force);
       } else if (i == TSDB_RETENTION_L2) {
-        SMA_OPEN_RSMA_IMPL(pVnode, 2);
+        SMA_OPEN_RSMA_IMPL(pVnode, 2, force);
       }
     }
 

source/dnode/vnode/src/sma/smaSnapshot.c

@@ -48,7 +48,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead
   // open rsma1/rsma2
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pSma->pRSmaTsdb[i]) {
-      code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2,
+      code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, (i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2), NULL,
                                 &pReader->pDataReader[i]);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -128,7 +128,7 @@ struct SRSmaSnapWriter {
   STsdbSnapWriter* pDataWriter[TSDB_RETENTION_L2];
 };
 
-int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter) {
+int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, void** ppRanges, SRSmaSnapWriter** ppWriter) {
   int32_t          code = 0;
   int32_t          lino = 0;
   SVnode*          pVnode = pSma->pVnode;
@@ -147,7 +147,7 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit
   // rsma1/rsma2
   for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
     if (pSma->pRSmaTsdb[i]) {
-      code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, &pWriter->pDataWriter[i]);
+      code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, ((void**)ppRanges)[i], &pWriter->pDataWriter[i]);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
   }
@@ -165,6 +165,21 @@ _exit:
   return code;
 }
 
+int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) {
+  int32_t code = 0;
+  for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
+    if (pWriter->pDataWriter[i]) {
+      code = tsdbSnapWriterPrepareClose(pWriter->pDataWriter[i]);
+      if (code) {
+        smaError("vgId:%d, failed to prepare close tsdbSnapWriter since %s. i: %d", SMA_VID(pWriter->pSma), terrstr(),
+                 i);
+        return -1;
+      }
+    }
+  }
+  return code;
+}
+
 int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) {
   int32_t          code = 0;
   int32_t          lino = 0;

source/dnode/vnode/src/tq/tq.c

@@ -82,6 +82,9 @@ void tqDestroyTqHandle(void* data) {
     taosMemoryFree(pData->msg);
     pData->msg = NULL;
   }
+  if (pData->block != NULL){
+    blockDataDestroy(pData->block);
+  }
 }
 
 static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) {
@@ -1197,6 +1200,9 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
       pStreamTask->status.taskStatus = TASK_STATUS__HALT;
 
       nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
+      if (nextProcessedVer == -1) {
+        nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1;
+      }
 
       tqDebug("s-task:%s level:%d nextProcessedVer:%" PRId64 ", sched-status:%d is halt by fill-history task:%s",
               pStreamTask->id.idStr, pStreamTask->info.taskLevel, nextProcessedVer, pStreamTask->status.schedStatus,
@@ -1972,4 +1978,4 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
 
   streamMetaReleaseTask(pMeta, pTask);
   return TSDB_CODE_SUCCESS;
-}
+}

source/dnode/vnode/src/tq/tqOffset.c

@@ -37,11 +37,9 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
 
   int32_t vgId = TD_VID(pStore->pTq->pVnode);
   int64_t code = 0;
-
+  int32_t size = 0;
-  STqOffsetHead head = {0};
-
   while (1) {
-    if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) {
+    if ((code = taosReadFile(pFile, &size, INT_BYTES)) != INT_BYTES) {
       if (code == 0) {
         break;
       } else {
@@ -49,7 +47,6 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
       }
     }
 
-    int32_t size = htonl(head.size);
     void*   pMemBuf = taosMemoryCalloc(1, size);
     if (pMemBuf == NULL) {
       tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size);
@@ -175,11 +172,11 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
       return -1;
     }
 
-    int32_t totLen = sizeof(STqOffsetHead) + bodyLen;
+    int32_t totLen = INT_BYTES + bodyLen;
     void*   buf = taosMemoryCalloc(1, totLen);
-    void*   abuf = POINTER_SHIFT(buf, sizeof(STqOffsetHead));
+    void*   abuf = POINTER_SHIFT(buf, INT_BYTES);
 
-    ((STqOffsetHead*)buf)->size = htonl(bodyLen);
+    *(int32_t*)buf = bodyLen;
     SEncoder encoder;
     tEncoderInit(&encoder, abuf, bodyLen);
     tEncodeSTqOffset(&encoder, pOffset);

source/dnode/vnode/src/tq/tqRead.c

@@ -369,82 +369,56 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
 // todo ignore the error in wal?
 bool tqNextBlockInWal(STqReader* pReader, const char* id) {
   SWalReader* pWalReader = pReader->pWalReader;
+  SSDataBlock* pDataBlock = NULL;
 
   uint64_t st = taosGetTimestampMs();
   while (1) {
-    SArray* pBlockList = pReader->submit.aSubmitTbData;
+    // try next message in wal file
-    if (pBlockList == NULL || pReader->nextBlk >= taosArrayGetSize(pBlockList)) {
+    if (walNextValidMsg(pWalReader) < 0) {
-      // try next message in wal file
+      return false;
-      // todo always retry to avoid read failure caused by wal file deletion
-      if (walNextValidMsg(pWalReader) < 0) {
-        return false;
-      }
-
-      void*   pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
-      int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
-      int64_t ver = pWalReader->pHead->head.version;
-
-      SDecoder decoder = {0};
-      tDecoderInit(&decoder, pBody, bodyLen);
-
-      {
-        int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData);
-        for (int32_t i = 0; i < nSubmitTbData; i++) {
-          SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i);
-          if (pData->pCreateTbReq != NULL) {
-            taosArrayDestroy(pData->pCreateTbReq->ctb.tagName);
-            taosMemoryFreeClear(pData->pCreateTbReq);
-          }
-          pData->aRowP = taosArrayDestroy(pData->aRowP);
-        }
-        pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData);
-      }
-
-      if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) {
-        tDecoderClear(&decoder);
-        tqError("decode wal file error, msgLen:%d, ver:%" PRId64, bodyLen, ver);
-        return false;
-      }
-
-      tDecoderClear(&decoder);
-      pReader->nextBlk = 0;
     }
 
+    void*   pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg));
+    int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg);
+    int64_t ver = pWalReader->pHead->head.version;
+
+    tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver);
+    pReader->nextBlk = 0;
     int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData);
     while (pReader->nextBlk < numOfBlocks) {
-      tqTrace("tq reader next data block %d/%d, len:%d %" PRId64 " %d", pReader->nextBlk,
+      tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk,
-          numOfBlocks, pReader->msg.msgLen, pReader->msg.ver, pReader->nextBlk);
+          numOfBlocks, pReader->msg.msgLen, pReader->msg.ver);
 
       SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk);
 
-      if (pReader->tbIdHash == NULL) {
+      if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) {
-        SSDataBlock* pRes = NULL;
+        tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid);
-        int32_t      code = tqRetrieveDataBlock(pReader, &pRes, NULL);
-        if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
-          return true;
-        }
-      }
-
-      void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t));
-      if (ret != NULL) {
-        tqTrace("tq reader return submit block, uid:%" PRId64 ", ver:%" PRId64, pSubmitTbData->uid, pReader->msg.ver);
-
         SSDataBlock* pRes = NULL;
         int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL);
         if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) {
-          return true;
+          if(pDataBlock == NULL){
+            pDataBlock = createOneDataBlock(pRes, true);
+          }else{
+            blockDataMerge(pDataBlock, pRes);
+          }
         }
       } else {
         pReader->nextBlk += 1;
         tqTrace("tq reader discard submit block, uid:%" PRId64 ", continue", pSubmitTbData->uid);
       }
     }
-
-    qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id);
     tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE);
-
     pReader->msg.msgStr = NULL;
 
+    if(pDataBlock != NULL){
+      blockDataCleanup(pReader->pResBlock);
+      copyDataBlock(pReader->pResBlock, pDataBlock);
+      blockDataDestroy(pDataBlock);
+      return true;
+    }else{
+      qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id);
+    }
+
     if(taosGetTimestampMs() - st > 1000){
       return false;
     }
@@ -478,6 +452,10 @@ SSDataBlock* tqGetResultBlock (STqReader* pReader) {
   return pReader->pResBlock;
 }
 
+int64_t tqGetResultBlockTime(STqReader *pReader){
+  return pReader->lastTs;
+}
+
 bool tqNextBlockImpl(STqReader* pReader, const char* idstr) {
   if (pReader->msg.msgStr == NULL) {
     return false;
@@ -644,7 +622,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char*
   int32_t sversion = pSubmitTbData->sver;
   int64_t suid = pSubmitTbData->suid;
   int64_t uid = pSubmitTbData->uid;
-  pReader->lastBlkUid = uid;
+  pReader->lastTs = pSubmitTbData->ctimeMs;
 
   pBlock->info.id.uid = uid;
   pBlock->info.version = pReader->msg.ver;
@@ -786,7 +764,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
   }
 
   int32_t sversion = pSubmitTbData->sver;
-  int64_t suid = pSubmitTbData->suid;
   int64_t uid = pSubmitTbData->uid;
   pReader->lastBlkUid = uid;
 

source/dnode/vnode/src/tq/tqScan.c

@@ -64,7 +64,23 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in
   return 0;
 }
 
-int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) {
+int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res){
+  uint64_t     ts = 0;
+  qStreamSetOpen(task);
+
+  tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId);
+  int32_t code = qExecTask(task, res, &ts);
+  if (code != TSDB_CODE_SUCCESS) {
+    tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code));
+    terrno = code;
+    return -1;
+  }
+
+  tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, *res);
+  return 0;
+}
+
+int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset, const SMqPollReq* pRequest) {
   const int32_t MAX_ROWS_TO_RETURN = 4096;
 
   int32_t vgId = TD_VID(pTq->pVnode);
@@ -80,34 +96,66 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
 
   while (1) {
     SSDataBlock* pDataBlock = NULL;
-    uint64_t     ts = 0;
+    code = getDataBlock(task, pHandle, vgId, &pDataBlock);
-    qStreamSetOpen(task);
+    if (code != 0){
-
-    tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId);
-    code = qExecTask(task, &pDataBlock, &ts);
-    if (code != TSDB_CODE_SUCCESS) {
-      tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, tstrerror(code));
-      terrno = code;
-      return -1;
-    }
-
-    tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId,
-            pDataBlock);
-    // current scan should be stopped asap, since the rebalance occurs.
-    if (pDataBlock == NULL) {
-      break;
-    }
-
-    code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
-    if (code != TSDB_CODE_SUCCESS) {
-      tqError("vgId:%d, failed to add block to rsp msg", vgId);
       return code;
     }
 
-    pRsp->blockNum++;
+    if(pRequest->enableReplay){
-    totalRows += pDataBlock->info.rows;
+      if(IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL){
-    if (totalRows >= MAX_ROWS_TO_RETURN) {
+        blockDataDestroy(pHandle->block);
+        pHandle->block = NULL;
+      }
+      if(pHandle->block == NULL){
+        if (pDataBlock == NULL) {
+          break;
+        }
+        STqOffsetVal offset = {0};
+        qStreamExtractOffset(task, &offset);
+        pHandle->block = createOneDataBlock(pDataBlock, true);
+//        pHandle->block = createDataBlock();
+//        copyDataBlock(pHandle->block, pDataBlock);
+        pHandle->blockTime = offset.ts;
+        code = getDataBlock(task, pHandle, vgId, &pDataBlock);
+        if (code != 0){
+          return code;
+        }
+      }
+
+      code = tqAddBlockDataToRsp(pHandle->block, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
+      if (code != TSDB_CODE_SUCCESS) {
+        tqError("vgId:%d, failed to add block to rsp msg", vgId);
+        return code;
+      }
+
+      pRsp->blockNum++;
+      if (pDataBlock == NULL) {
+        blockDataDestroy(pHandle->block);
+        pHandle->block = NULL;
+      }else{
+        copyDataBlock(pHandle->block, pDataBlock);
+
+        STqOffsetVal offset = {0};
+        qStreamExtractOffset(task, &offset);
+        pRsp->sleepTime = offset.ts - pHandle->blockTime;
+        pHandle->blockTime = offset.ts;
+      }
       break;
+    }else{
+      if (pDataBlock == NULL) {
+        break;
+      }
+      code = tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision);
+      if (code != TSDB_CODE_SUCCESS) {
+        tqError("vgId:%d, failed to add block to rsp msg", vgId);
+        return code;
+      }
+
+      pRsp->blockNum++;
+      totalRows += pDataBlock->info.rows;
+      if (totalRows >= MAX_ROWS_TO_RETURN) {
+        break;
+      }
     }
   }
 

source/dnode/vnode/src/tq/tqStreamTaskSnap.c

@@ -198,8 +198,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) {
 
   taosWLockLatch(&pTq->pStreamMeta->lock);
   tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode));
-
-  taosWLockLatch(&pTq->pStreamMeta->lock);
   if (rollback) {
     tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);
   } else {
@@ -208,12 +206,6 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) {
     code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);
     if (code) goto _err;
   }
-  if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
-    code = -1;
-    goto _err;
-  }
-
-  taosWUnLockLatch(&pTq->pStreamMeta->lock);
 
   if (tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) {
     code = -1;

source/dnode/vnode/src/tq/tqUtil.c

@@ -15,8 +15,6 @@
 
 #include "tq.h"
 
-#define IS_OFFSET_RESET_TYPE(_t)  ((_t) < 0)
-
 static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq,
                                  const SMqMetaRsp* pRsp, int32_t vgId);
 
@@ -152,7 +150,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
   tqInitDataRsp(&dataRsp, *pOffset);
 
   qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId);
-  int code = tqScanData(pTq, pHandle, &dataRsp, pOffset);
+  int code = tqScanData(pTq, pHandle, &dataRsp, pOffset, pRequest);
   if (code != 0 && terrno != TSDB_CODE_WAL_LOG_NOT_EXIST) {
     goto end;
   }

source/dnode/vnode/src/tsdb/tsdbDataFileRW.c

@@ -15,10 +15,6 @@
 
 #include "tsdbDataFileRW.h"
 
-extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
-                                      TTombBlkArray *tombBlkArray, uint8_t **bufArr);
-extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize);
-
 // SDataFileReader =============================================
 struct SDataFileReader {
   SDataFileReaderConfig config[1];
@@ -491,6 +487,9 @@ struct SDataFileWriter {
     int32_t              tombBlkArrayIdx;
     STombBlock           tombBlock[1];
     int32_t              tombBlockIdx;
+    // range
+    SVersionRange        range;
+    SVersionRange        tombRange;
   } ctx[1];
 
   STFile   files[TSDB_FTYPE_MAX];
@@ -589,6 +588,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
       .fid = writer->config->fid,
       .cid = writer->config->cid,
       .size = 0,
+      .minVer = VERSION_MAX,
+      .maxVer = VERSION_MIN,
   };
 
   // .data
@@ -602,6 +603,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
         .fid = writer->config->fid,
         .cid = writer->config->cid,
         .size = 0,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
   }
 
@@ -616,6 +619,8 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
         .fid = writer->config->fid,
         .cid = writer->config->cid,
         .size = 0,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
   }
 
@@ -627,8 +632,14 @@ static int32_t tsdbDataFileWriterDoOpen(SDataFileWriter *writer) {
       .fid = writer->config->fid,
       .cid = writer->config->cid,
       .size = 0,
+      .minVer = VERSION_MAX,
+      .maxVer = VERSION_MIN,
   };
 
+  // range
+  writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+  writer->ctx->tombRange = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+
   writer->ctx->opened = true;
 
 _exit:
@@ -638,8 +649,14 @@ _exit:
   return code;
 }
 
+int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer) {
+  range->minVer = TMIN(range->minVer, minVer);
+  range->maxVer = TMAX(range->maxVer, maxVer);
+  return 0;
+}
+
 int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize,
-                               TBrinBlkArray *brinBlkArray, uint8_t **bufArr) {
+                               TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range) {
   if (BRIN_BLOCK_SIZE(brinBlock) == 0) return 0;
 
   int32_t code;
@@ -678,6 +695,8 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAl
     }
   }
 
+  tsdbWriterUpdVerRange(range, brinBlk->minVer, brinBlk->maxVer);
+
   // write to file
   for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->dataArr1); i++) {
     code = tsdbCmprData((uint8_t *)TARRAY2_DATA(brinBlock->dataArr1 + i), TARRAY2_DATA_LEN(brinBlock->dataArr1 + i),
@@ -728,7 +747,8 @@ static int32_t tsdbDataFileWriteBrinBlock(SDataFileWriter *writer) {
   int32_t lino = 0;
 
   code = tsdbFileWriteBrinBlock(writer->fd[TSDB_FTYPE_HEAD], writer->brinBlock, writer->config->cmprAlg,
-                                &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr);
+                                &writer->files[TSDB_FTYPE_HEAD].size, writer->brinBlkArray, writer->config->bufArr,
+                                &writer->ctx->range);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -795,6 +815,8 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
     }
   }
 
+  tsdbWriterUpdVerRange(&writer->ctx->range, record->minVer, record->maxVer);
+
   // to .data file
   int32_t sizeArr[5] = {0};
 
@@ -1143,6 +1165,64 @@ int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFoote
   return 0;
 }
 
+int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
+                               TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range) {
+  int32_t code;
+
+  if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0;
+
+  STombBlk tombBlk[1] = {{
+      .dp[0] =
+          {
+              .offset = *fileSize,
+              .size = 0,
+          },
+      .minTbid =
+          {
+              .suid = TARRAY2_FIRST(tombBlock->suid),
+              .uid = TARRAY2_FIRST(tombBlock->uid),
+          },
+      .maxTbid =
+          {
+              .suid = TARRAY2_LAST(tombBlock->suid),
+              .uid = TARRAY2_LAST(tombBlock->uid),
+          },
+      .minVer = TARRAY2_FIRST(tombBlock->version),
+      .maxVer = TARRAY2_FIRST(tombBlock->version),
+      .numRec = TOMB_BLOCK_SIZE(tombBlock),
+      .cmprAlg = cmprAlg,
+  }};
+
+  for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) {
+    if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) {
+      tombBlk->minVer = TARRAY2_GET(tombBlock->version, i);
+    }
+    if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) {
+      tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i);
+    }
+  }
+
+  tsdbWriterUpdVerRange(range, tombBlk->minVer, tombBlk->maxVer);
+
+  for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) {
+    code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]),
+                        TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]);
+    if (code) return code;
+
+    code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]);
+    if (code) return code;
+
+    tombBlk->dp->size += tombBlk->size[i];
+    *fileSize += tombBlk->size[i];
+  }
+
+  code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk);
+  if (code) return code;
+
+  tTombBlockClear(tombBlock);
+  return 0;
+}
+
 static int32_t tsdbDataFileWriteHeadFooter(SDataFileWriter *writer) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -1164,7 +1244,8 @@ static int32_t tsdbDataFileDoWriteTombBlock(SDataFileWriter *writer) {
   int32_t lino = 0;
 
   code = tsdbFileWriteTombBlock(writer->fd[TSDB_FTYPE_TOMB], writer->tombBlock, writer->config->cmprAlg,
-                                &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr);
+                                &writer->files[TSDB_FTYPE_TOMB].size, writer->tombBlkArray, writer->config->bufArr,
+                                &writer->ctx->tombRange);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -1174,6 +1255,21 @@ _exit:
   return code;
 }
 
+int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
+  ptr->size = TARRAY2_DATA_LEN(tombBlkArray);
+  if (ptr->size > 0) {
+    ptr->offset = *fileSize;
+
+    int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size);
+    if (code) {
+      return code;
+    }
+
+    *fileSize += ptr->size;
+  }
+  return 0;
+}
+
 static int32_t tsdbDataFileDoWriteTombBlk(SDataFileWriter *writer) {
   ASSERT(TARRAY2_SIZE(writer->tombBlkArray) > 0);
 
@@ -1306,6 +1402,12 @@ _exit:
   return code;
 }
 
+int32_t tsdbTFileUpdVerRange(STFile *f, SVersionRange range) {
+  f->minVer = TMIN(f->minVer, range.minVer);
+  f->maxVer = TMAX(f->maxVer, range.maxVer);
+  return 0;
+}
+
 static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArray *opArr) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -1334,6 +1436,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
     code = tsdbDataFileWriteHeadFooter(writer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
+    SVersionRange ofRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+
     // .head
     ftype = TSDB_FTYPE_HEAD;
     if (writer->config->files[ftype].exist) {
@@ -1342,6 +1446,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .fid = writer->config->fid,
           .of = writer->config->files[ftype].file,
       };
+      ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer};
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -1350,6 +1455,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
         .fid = writer->config->fid,
         .nf = writer->files[ftype],
     };
+    tsdbTFileUpdVerRange(&op.nf, ofRange);
+    tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
     code = TARRAY2_APPEND(opArr, op);
     TSDB_CHECK_CODE(code, lino, _exit);
 
@@ -1361,6 +1468,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .fid = writer->config->fid,
           .nf = writer->files[ftype],
       };
+      tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) {
@@ -1370,6 +1478,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .of = writer->config->files[ftype].file,
           .nf = writer->files[ftype],
       };
+      tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -1382,6 +1491,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .fid = writer->config->fid,
           .nf = writer->files[ftype],
       };
+      tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     } else if (writer->config->files[ftype].file.size != writer->files[ftype].size) {
@@ -1391,6 +1501,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .of = writer->config->files[ftype].file,
           .nf = writer->files[ftype],
       };
+      tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -1415,6 +1526,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
     code = tsdbDataFileWriteTombFooter(writer);
     TSDB_CHECK_CODE(code, lino, _exit);
 
+    SVersionRange ofRange = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+
     ftype = TSDB_FTYPE_TOMB;
     if (writer->config->files[ftype].exist) {
       op = (STFileOp){
@@ -1422,6 +1535,7 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
           .fid = writer->config->fid,
           .of = writer->config->files[ftype].file,
       };
+      ofRange = (SVersionRange){.minVer = op.of.minVer, .maxVer = op.of.maxVer};
       code = TARRAY2_APPEND(opArr, op);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
@@ -1430,6 +1544,8 @@ static int32_t tsdbDataFileWriterCloseCommit(SDataFileWriter *writer, TFileOpArr
         .fid = writer->config->fid,
         .nf = writer->files[ftype],
     };
+    tsdbTFileUpdVerRange(&op.nf, ofRange);
+    tsdbTFileUpdVerRange(&op.nf, writer->ctx->tombRange);
     code = TARRAY2_APPEND(opArr, op);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
@@ -1598,6 +1714,7 @@ int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData) {
   ) {
     code = tsdbDataFileDoWriteBlockData(writer, bData);
     TSDB_CHECK_CODE(code, lino, _exit);
+
   } else {
     for (int32_t i = 0; i < bData->nRow; ++i) {
       TSDBROW row[1] = {tsdbRowFromBlockData(bData, i)};

source/dnode/vnode/src/tsdb/tsdbDataFileRW.h

@@ -95,10 +95,25 @@ int32_t tsdbDataFileWriteRow(SDataFileWriter *writer, SRowInfo *row);
 int32_t tsdbDataFileWriteBlockData(SDataFileWriter *writer, SBlockData *bData);
 int32_t tsdbDataFileFlush(SDataFileWriter *writer);
 
+// head
+int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize,
+                               TBrinBlkArray *brinBlkArray, uint8_t **bufArr, SVersionRange *range);
+int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize);
+int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer);
+
+// tomb
 int32_t tsdbDataFileWriteTombRecord(SDataFileWriter *writer, const STombRecord *record);
+int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
+                               TTombBlkArray *tombBlkArray, uint8_t **bufArr, SVersionRange *range);
+int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize);
+int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize);
+
+// utils
+int32_t tsdbWriterUpdVerRange(SVersionRange *range, int64_t minVer, int64_t maxVer);
+int32_t tsdbTFileUpdVerRange(STFile *f, SVersionRange range);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TSDB_DATA_FILE_RW_H*/
+#endif /*_TSDB_DATA_FILE_RW_H*/

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -38,13 +38,6 @@ typedef struct {
   STFileHashEntry **buckets;
 } STFileHash;
 
-enum {
-  TSDB_FS_STATE_NONE = 0,
-  TSDB_FS_STATE_OPEN,
-  TSDB_FS_STATE_EDIT,
-  TSDB_FS_STATE_CLOSE,
-};
-
 static const char *gCurrentFname[] = {
     [TSDB_FCURRENT] = "current.json",
     [TSDB_FCURRENT_C] = "current.c.json",
@@ -57,7 +50,7 @@ static int32_t create_fs(STsdb *pTsdb, STFileSystem **fs) {
 
   fs[0]->tsdb = pTsdb;
   tsem_init(&fs[0]->canEdit, 0, 1);
-  fs[0]->state = TSDB_FS_STATE_NONE;
+  fs[0]->fsstate = TSDB_FS_STATE_NORMAL;
   fs[0]->neid = 0;
   TARRAY2_INIT(fs[0]->fSetArr);
   TARRAY2_INIT(fs[0]->fSetArrTmp);
@@ -258,14 +251,6 @@ _exit:
   return code;
 }
 
-static bool is_same_file(const STFile *f1, const STFile f2) {
-  if (f1->type != f2.type) return false;
-  if (f1->did.level != f2.did.level) return false;
-  if (f1->did.id != f2.did.id) return false;
-  if (f1->cid != f2.cid) return false;
-  return true;
-}
-
 static int32_t apply_commit(STFileSystem *fs) {
   int32_t        code = 0;
   TFileSetArray *fsetArray1 = fs->fSetArr;
@@ -504,6 +489,7 @@ static void tsdbFSDestroyFileObjHash(STFileHash *hash) {
 static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
   int32_t code = 0;
   int32_t lino = 0;
+  int32_t corrupt = false;
 
   {  // scan each file
     STFileSet *fset = NULL;
@@ -511,8 +497,12 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
       // data file
       for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) {
         if (fset->farr[ftype] == NULL) continue;
-        code = tsdbFSDoScanAndFixFile(fs, fset->farr[ftype]);
+        STFileObj *fobj = fset->farr[ftype];
-        TSDB_CHECK_CODE(code, lino, _exit);
+        code = tsdbFSDoScanAndFixFile(fs, fobj);
+        if (code) {
+          fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
+          corrupt = true;
+        }
       }
 
       // stt file
@@ -521,12 +511,22 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
         STFileObj *fobj;
         TARRAY2_FOREACH(lvl->fobjArr, fobj) {
           code = tsdbFSDoScanAndFixFile(fs, fobj);
-          TSDB_CHECK_CODE(code, lino, _exit);
+          if (code) {
+            fset->maxVerValid = (fobj->f->minVer <= fobj->f->maxVer) ? TMIN(fset->maxVerValid, fobj->f->minVer - 1) : -1;
+            corrupt = true;
+          }
         }
       }
     }
   }
 
+  if (corrupt) {
+    tsdbError("vgId:%d, not to clear dangling files due to fset incompleteness", TD_VID(fs->tsdb->pVnode));
+    fs->fsstate = TSDB_FS_STATE_INCOMPLETE;
+    code = 0;
+    goto _exit;
+  }
+
   {  // clear unreferenced files
     STfsDir *dir = tfsOpendir(fs->tsdb->pVnode->pTfs, fs->tsdb->path);
     if (dir == NULL) {
@@ -961,6 +961,13 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) {
 }
 
 int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
+  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr);
+  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+  return code;
+}
+
+int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr) {
   int32_t    code = 0;
   STFileSet *fset, *fset1;
 
@@ -991,6 +998,142 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) {
   return 0;
 }
 
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr,
+                                       TFileOpArray *fopArr) {
+  int32_t    code = 0;
+  STFileSet *fset;
+  STFileSet *fset1;
+  SHashObj  *pHash = NULL;
+
+  fsetArr[0] = taosMemoryMalloc(sizeof(TFileSetArray));
+  if (fsetArr == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  TARRAY2_INIT(fsetArr[0]);
+
+  if (pRanges) {
+    pHash = tsdbGetSnapRangeHash(pRanges);
+    if (pHash == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _out;
+    }
+  }
+
+  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  TARRAY2_FOREACH(fs->fSetArr, fset) {
+    int64_t ever = VERSION_MAX;
+    if (pHash) {
+      int32_t      fid = fset->fid;
+      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      if (u) {
+        ever = u->sver - 1;
+      }
+    }
+
+    code = tsdbTFileSetFilteredInitDup(fs->tsdb, fset, ever, &fset1, fopArr);
+    if (code) break;
+
+    code = TARRAY2_APPEND(fsetArr[0], fset1);
+    if (code) break;
+  }
+  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+
+_out:
+  if (code) {
+    TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear);
+    taosMemoryFree(fsetArr[0]);
+    fsetArr[0] = NULL;
+  }
+  if (pHash) {
+    taosHashCleanup(pHash);
+    pHash = NULL;
+  }
+  return code;
+}
+
+SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) {
+  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
+  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+  if (pHash == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
+    STSnapRange *u = TARRAY2_GET(pRanges, i);
+    int32_t      fid = u->fid;
+    int32_t      code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
+    ASSERT(code == 0);
+    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
+  }
+  return pHash;
+}
+
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
+                                      TSnapRangeArray **fsrArr) {
+  int32_t      code = 0;
+  STFileSet   *fset;
+  STSnapRange *fsr1 = NULL;
+  SHashObj    *pHash = NULL;
+
+  fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0]));
+  if (fsrArr[0] == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _out;
+  }
+
+  tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges)));
+  if (pRanges) {
+    pHash = tsdbGetSnapRangeHash(pRanges);
+    if (pHash == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _out;
+    }
+  }
+
+  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  TARRAY2_FOREACH(fs->fSetArr, fset) {
+    int64_t sver1 = sver;
+    int64_t ever1 = ever;
+
+    if (pHash) {
+      int32_t      fid = fset->fid;
+      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      if (u) {
+        sver1 = u->sver;
+        tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
+      }
+    }
+
+    if (sver1 > ever1) {
+      tsdbDebug("skip fid:%d, sver:%" PRId64 ", ever:%" PRId64, fset->fid, sver1, ever1);
+      continue;
+    }
+
+    tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1);
+
+    code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
+    if (code) break;
+
+    code = TARRAY2_APPEND(fsrArr[0], fsr1);
+    if (code) break;
+
+    fsr1 = NULL;
+  }
+  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+
+  if (code) {
+    tsdbTSnapRangeClear(&fsr1);
+    TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear);
+    fsrArr[0] = NULL;
+  }
+
+_out:
+  if (pHash) {
+    taosHashCleanup(pHash);
+    pHash = NULL;
+  }
+  return code;
+}
+
 const char *gFSBgTaskName[] = {NULL, "MERGE", "RETENTION", "COMPACT"};
 
 static int32_t tsdbFSRunBgTask(void *arg) {
@@ -1148,4 +1291,4 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs) {
   fs->stop = false;
   taosThreadMutexUnlock(fs->mutex);
   return 0;
-}
+}

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -52,7 +52,15 @@ int32_t tsdbCloseFS(STFileSystem **fs);
 int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr);
 int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr);
+int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr);
+
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr,
+                                       TFileOpArray *fopArr);
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr);
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
+                                      TSnapRangeArray **fsrArr);
+int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr);
 // txn
 int64_t tsdbFSAllocEid(STFileSystem *fs);
 int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype);
@@ -68,6 +76,9 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs);
 // other
 int32_t tsdbFSGetFSet(STFileSystem *fs, int32_t fid, STFileSet **fset);
 int32_t tsdbFSCheckCommit(STFileSystem *fs);
+// utils
+int32_t save_fs(const TFileSetArray *arr, const char *fname);
+int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
 
 struct STFSBgTask {
   EFSBgTaskT type;
@@ -91,7 +102,7 @@ struct STFSBgTask {
 struct STFileSystem {
   STsdb        *tsdb;
   tsem_t        canEdit;
-  int32_t       state;
+  int32_t       fsstate;
   int64_t       neid;
   EFEditT       etype;
   TFileSetArray fSetArr[1];

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -65,6 +65,34 @@ static int32_t tsdbSttLvlInitRef(STsdb *pTsdb, const SSttLvl *lvl1, SSttLvl **lv
   return 0;
 }
 
+static int32_t tsdbSttLvlFilteredInitEx(STsdb *pTsdb, const SSttLvl *lvl1, int64_t ever, SSttLvl **lvl,
+                                        TFileOpArray *fopArr) {
+  int32_t code = tsdbSttLvlInit(lvl1->level, lvl);
+  if (code) return code;
+
+  const STFileObj *fobj1;
+  TARRAY2_FOREACH(lvl1->fobjArr, fobj1) {
+    if (fobj1->f->maxVer <= ever) {
+      STFileObj *fobj;
+      code = tsdbTFileObjInit(pTsdb, fobj1->f, &fobj);
+      if (code) {
+        tsdbSttLvlClear(lvl);
+        return code;
+      }
+
+      TARRAY2_APPEND(lvl[0]->fobjArr, fobj);
+    } else {
+      STFileOp op = {
+          .optype = TSDB_FOP_REMOVE,
+          .fid = fobj1->f->fid,
+          .of = fobj1->f[0],
+      };
+      TARRAY2_APPEND(fopArr, op);
+    }
+  }
+  return 0;
+}
+
 static void tsdbSttLvlRemoveFObj(void *data) { tsdbTFileObjRemove(*(STFileObj **)data); }
 static void tsdbSttLvlRemove(SSttLvl **lvl) {
   TARRAY2_DESTROY(lvl[0]->fobjArr, tsdbSttLvlRemoveFObj);
@@ -424,6 +452,7 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
   if (fset[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
 
   fset[0]->fid = fid;
+  fset[0]->maxVerValid = VERSION_MAX;
   TARRAY2_INIT(fset[0]->lvlArr);
   return 0;
 }
@@ -458,6 +487,61 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
   return 0;
 }
 
+int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
+                                    TFileOpArray *fopArr) {
+  int32_t code = tsdbTFileSetInit(fset1->fid, fset);
+  if (code) return code;
+
+  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+    if (fset1->farr[ftype] == NULL) continue;
+    STFileObj *fobj = fset1->farr[ftype];
+    if (fobj->f->maxVer <= ever) {
+      code = tsdbTFileObjInit(pTsdb, fobj->f, &fset[0]->farr[ftype]);
+      if (code) {
+        tsdbTFileSetClear(fset);
+        return code;
+      }
+    } else {
+      STFileOp op = {
+          .optype = TSDB_FOP_REMOVE,
+          .fid = fobj->f->fid,
+          .of = fobj->f[0],
+      };
+      TARRAY2_APPEND(fopArr, op);
+    }
+  }
+
+  const SSttLvl *lvl1;
+  TARRAY2_FOREACH(fset1->lvlArr, lvl1) {
+    SSttLvl *lvl;
+    code = tsdbSttLvlFilteredInitEx(pTsdb, lvl1, ever, &lvl, fopArr);
+    if (code) {
+      tsdbTFileSetClear(fset);
+      return code;
+    }
+
+    code = TARRAY2_APPEND(fset[0]->lvlArr, lvl);
+    if (code) return code;
+  }
+
+  return 0;
+}
+
+int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) {
+  fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0]));
+  if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+  fsr[0]->fid = fset1->fid;
+  fsr[0]->sver = sver;
+  fsr[0]->ever = ever;
+
+  int32_t code = tsdbTFileSetInitRef(pTsdb, fset1, &fsr[0]->fset);
+  if (code) {
+    taosMemoryFree(fsr[0]);
+    fsr[0] = NULL;
+  }
+  return code;
+}
+
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset) {
   int32_t code = tsdbTFileSetInit(fset1->fid, fset);
   if (code) return code;
@@ -485,6 +569,15 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
   return 0;
 }
 
+int32_t tsdbTSnapRangeClear(STSnapRange **fsr) {
+  if (!fsr[0]) return 0;
+
+  tsdbTFileSetClear(&fsr[0]->fset);
+  taosMemoryFree(fsr[0]);
+  fsr[0] = NULL;
+  return 0;
+}
+
 int32_t tsdbTFileSetClear(STFileSet **fset) {
   if (!fset[0]) return 0;
 
@@ -545,4 +638,4 @@ bool tsdbTFileSetIsEmpty(const STFileSet *fset) {
     if (fset->farr[ftype] != NULL) return false;
   }
   return TARRAY2_SIZE(fset->lvlArr) == 0;
-}
+}

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -45,6 +45,13 @@ int32_t tsdbTFileSetInitDup(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
 int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fset);
 int32_t tsdbTFileSetClear(STFileSet **fset);
 int32_t tsdbTFileSetRemove(STFileSet **fset);
+
+int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
+                                    TFileOpArray *fopArr);
+
+int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr);
+int32_t tsdbTSnapRangeClear(STSnapRange **fsr);
+
 // to/from json
 int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json);
 int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset);
@@ -59,6 +66,9 @@ int64_t tsdbTFileSetMaxCid(const STFileSet *fset);
 SSttLvl *tsdbTFileSetGetSttLvl(STFileSet *fset, int32_t level);
 // is empty
 bool tsdbTFileSetIsEmpty(const STFileSet *fset);
+// stt
+int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl);
+int32_t tsdbSttLvlClear(SSttLvl **lvl);
 
 struct STFileOp {
   tsdb_fop_t optype;
@@ -74,12 +84,20 @@ struct SSttLvl {
 
 struct STFileSet {
   int32_t      fid;
+  int64_t      maxVerValid;
   STFileObj   *farr[TSDB_FTYPE_MAX];  // file array
   TSttLvlArray lvlArr[1];             // level array
 };
 
+struct STSnapRange {
+  int32_t    fid;
+  int64_t    sver;
+  int64_t    ever;
+  STFileSet *fset;
+};
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TSDB_FILE_SET2_H*/
+#endif /*_TSDB_FILE_SET2_H*/

source/dnode/vnode/src/tsdb/tsdbFSetRW.c

@@ -292,4 +292,4 @@ _exit:
     TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
   }
   return code;
-}
+}

source/dnode/vnode/src/tsdb/tsdbFSetRW.h

@@ -52,4 +52,4 @@ int32_t tsdbFSetWriteTombRecord(SFSetWriter *writer, const STombRecord *tombReco
 }
 #endif
 
-#endif /*_TSDB_FSET_RW_H*/
+#endif /*_TSDB_FSET_RW_H*/

source/dnode/vnode/src/tsdb/tsdbFile2.c

@@ -76,6 +76,17 @@ static int32_t tfile_to_json(const STFile *file, cJSON *json) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
 
+  if (file->minVer <= file->maxVer) {
+    /* minVer */
+    if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    /* maxVer */
+    if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+  }
   return 0;
 }
 
@@ -122,6 +133,19 @@ static int32_t tfile_from_json(const cJSON *json, STFile *file) {
     return TSDB_CODE_FILE_CORRUPTED;
   }
 
+  /* minVer */
+  file->minVer = VERSION_MAX;
+  item = cJSON_GetObjectItem(json, "minVer");
+  if (cJSON_IsNumber(item)) {
+    file->minVer = item->valuedouble;
+  }
+
+  /* maxVer */
+  file->maxVer = VERSION_MIN;
+  item = cJSON_GetObjectItem(json, "maxVer");
+  if (cJSON_IsNumber(item)) {
+    file->maxVer = item->valuedouble;
+  }
   return 0;
 }
 
@@ -296,4 +320,4 @@ int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) {
   } else {
     return 0;
   }
-}
+}

source/dnode/vnode/src/tsdb/tsdbFile2.h

@@ -61,6 +61,8 @@ struct STFile {
   int32_t      fid;  // file id
   int64_t      cid;  // commit id
   int64_t      size;
+  int64_t      minVer;
+  int64_t      maxVer;
   union {
     struct {
       int32_t level;
@@ -80,4 +82,4 @@ struct STFileObj {
 }
 #endif
 
-#endif /*_TSDB_FILE_H*/
+#endif /*_TSDB_FILE_H*/

source/dnode/vnode/src/tsdb/tsdbMerge.c

@@ -313,6 +313,7 @@ static int32_t tsdbMergeFileSetBeginOpenWriter(SMerger *merger) {
       if (merger->ctx->fset->farr[ftype]) {
         config.files[ftype].exist = true;
         config.files[ftype].file = merger->ctx->fset->farr[ftype]->f[0];
+
       } else {
         config.files[ftype].exist = false;
       }

source/dnode/vnode/src/tsdb/tsdbOpen.c

@@ -35,7 +35,7 @@ int32_t tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) {
  * @param dir
  * @return int
  */
-int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback) {
+int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
   STsdb *pTsdb = NULL;
   int    slen = 0;
 
@@ -72,6 +72,11 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee
     goto _err;
   }
 
+  if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) {
+    terrno = TSDB_CODE_NEED_RETRY;
+    goto _err;
+  }
+
   if (tsdbOpenCache(pTsdb) < 0) {
     goto _err;
   }

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -2670,6 +2670,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   STableBlockScanInfo* pScanInfo = NULL;
   SFileDataBlockInfo*  pBlockInfo = getCurrentBlockInfo(pBlockIter);
   SLastBlockReader*    pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
+  bool                 asc = ASCENDING_TRAVERSE(pReader->info.order);
 
   if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
     setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
@@ -2705,8 +2706,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   } else {
     bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
     int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
-    if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) ||
+    if (!bHasDataInLastBlock ||
-                                 (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) {
+        ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) {
       // whole block is required, return it directly
       SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
       pInfo->rows = pBlockInfo->record.numRow;
@@ -2728,26 +2729,28 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       tBlockDataReset(pBData);
 
       SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
-      tsdbDebug("load data in last block firstly %s", pReader->idStr);
 
+      tsdbDebug("load data in last block firstly %s", pReader->idStr);
       int64_t st = taosGetTimestampUs();
 
-      while (1) {
+      // no data in last block, no need to proceed.
-        bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+      while (hasDataInLastBlock(pLastBlockReader)) {
-
-        // no data in last block and block, no need to proceed.
-        if (hasBlockLData == false) {
-          break;
-        }
-
         code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
-        if (code) {
+        if (code != TSDB_CODE_SUCCESS) {
           return code;
         }
 
         if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {
           break;
         }
+
+        // data in stt now overlaps with current active file data block, need to composed with file data block.
+        int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+        if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) {
+          tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
+                    keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
+          break;
+        }
       }
 
       double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -2760,7 +2763,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
                   pResBlock->info.rows, el, pReader->idStr);
       }
     }
-
   }
 
   return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;
@@ -4947,7 +4949,7 @@ int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STs
   }
 
   // fs
-  code = tsdbFSCreateRefSnapshot(pTsdb->pFS, &pSnap->pfSetArray);
+  code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray);
 
   // unlock
   taosThreadRwlockUnlock(&pTsdb->rwLock);

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -151,6 +151,8 @@ static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const
               .type = fobj->f->type,
               .did = did[0],
               .fid = fobj->f->fid,
+              .minVer = fobj->f->minVer,
+              .maxVer = fobj->f->maxVer,
               .cid = fobj->f->cid,
               .size = fobj->f->size,
               .stt[0] =
@@ -198,6 +200,8 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const
               .type = fobj->f->type,
               .did = did[0],
               .fid = fobj->f->fid,
+              .minVer = fobj->f->minVer,
+              .maxVer = fobj->f->maxVer,
               .cid = fobj->f->cid,
               .size = fobj->f->size,
               .stt[0] =

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -32,12 +32,12 @@ struct STsdbSnapReader {
   uint8_t* aBuf[5];
   SSkmInfo skmTb[1];
 
-  TFileSetArray* fsetArr;
+  TSnapRangeArray* fsrArr;
 
   // context
   struct {
-    int32_t    fsetArrIdx;
+    int32_t      fsrArrIdx;
-    STFileSet* fset;
+    STSnapRange* fsr;
     bool       isDataDone;
     bool       isTombDone;
   } ctx[1];
@@ -72,10 +72,10 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
   };
   bool hasDataFile = false;
   for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) {
-    if (reader->ctx->fset->farr[ftype] != NULL) {
+    if (reader->ctx->fsr->fset->farr[ftype] != NULL) {
       hasDataFile = true;
       config.files[ftype].exist = true;
-      config.files[ftype].file = reader->ctx->fset->farr[ftype]->f[0];
+      config.files[ftype].file = reader->ctx->fsr->fset->farr[ftype]->f[0];
     }
   }
 
@@ -86,7 +86,7 @@ static int32_t tsdbSnapReadFileSetOpenReader(STsdbSnapReader* reader) {
 
   // stt
   SSttLvl* lvl;
-  TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) {
+  TARRAY2_FOREACH(reader->ctx->fsr->fset->lvlArr, lvl) {
     STFileObj* fobj;
     TARRAY2_FOREACH(lvl->fobjArr, fobj) {
       SSttFileReader*      sttReader;
@@ -138,8 +138,8 @@ static int32_t tsdbSnapReadFileSetOpenIter(STsdbSnapReader* reader) {
   STsdbIter*      iter;
   STsdbIterConfig config = {
       .filterByVersion = true,
-      .verRange[0] = reader->sver,
+      .verRange[0] = reader->ctx->fsr->sver,
-      .verRange[1] = reader->ever,
+      .verRange[1] = reader->ctx->fsr->ever,
   };
 
   // data file
@@ -211,14 +211,14 @@ static int32_t tsdbSnapReadFileSetCloseIter(STsdbSnapReader* reader) {
   return 0;
 }
 
-static int32_t tsdbSnapReadFileSetBegin(STsdbSnapReader* reader) {
+static int32_t tsdbSnapReadRangeBegin(STsdbSnapReader* reader) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  ASSERT(reader->ctx->fset == NULL);
+  ASSERT(reader->ctx->fsr == NULL);
 
-  if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) {
+  if (reader->ctx->fsrArrIdx < TARRAY2_SIZE(reader->fsrArr)) {
-    reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++);
+    reader->ctx->fsr = TARRAY2_GET(reader->fsrArr, reader->ctx->fsrArrIdx++);
     reader->ctx->isDataDone = false;
     reader->ctx->isTombDone = false;
 
@@ -236,10 +236,10 @@ _exit:
   return code;
 }
 
-static int32_t tsdbSnapReadFileSetEnd(STsdbSnapReader* reader) {
+static int32_t tsdbSnapReadRangeEnd(STsdbSnapReader* reader) {
   tsdbSnapReadFileSetCloseIter(reader);
   tsdbSnapReadFileSetCloseReader(reader);
-  reader->ctx->fset = NULL;
+  reader->ctx->fsr = NULL;
   return 0;
 }
 
@@ -412,7 +412,8 @@ _exit:
   return code;
 }
 
-int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, STsdbSnapReader** reader) {
+int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type, void* pRanges,
+                           STsdbSnapReader** reader) {
   int32_t code = 0;
   int32_t lino = 0;
 
@@ -424,22 +425,19 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type,
   reader[0]->ever = ever;
   reader[0]->type = type;
 
-  taosThreadRwlockRdlock(&tsdb->rwLock);
+  code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr);
-  code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr);
-  taosThreadRwlockUnlock(&tsdb->rwLock);
-
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
               __func__, lino, tstrerror(code), sver, ever, type);
-    tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+    tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
     taosMemoryFree(reader[0]);
     reader[0] = NULL;
   } else {
-    tsdbInfo("vgId:%d %s done, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__, sver, ever,
+    tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
-             type);
+             sver, ever, type);
   }
   return code;
 }
@@ -462,7 +460,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) {
   TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose);
   tsdbDataFileReaderClose(&reader[0]->dataReader);
 
-  tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+  tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
   tDestroyTSchema(reader[0]->skmTb->pTSchema);
 
   for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) {
@@ -488,11 +486,11 @@ int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) {
   data[0] = NULL;
 
   for (;;) {
-    if (reader->ctx->fset == NULL) {
+    if (reader->ctx->fsr == NULL) {
-      code = tsdbSnapReadFileSetBegin(reader);
+      code = tsdbSnapReadRangeBegin(reader);
       TSDB_CHECK_CODE(code, lino, _exit);
 
-      if (reader->ctx->fset == NULL) {
+      if (reader->ctx->fsr == NULL) {
         break;
       }
     }
@@ -517,7 +515,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* reader, uint8_t** data) {
       }
     }
 
-    code = tsdbSnapReadFileSetEnd(reader);
+    code = tsdbSnapReadRangeEnd(reader);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 
@@ -1030,7 +1028,7 @@ _exit:
   return code;
 }
 
-int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** writer) {
+int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRanges, STsdbSnapWriter** writer) {
   int32_t code = 0;
   int32_t lino = 0;
 
@@ -1054,7 +1052,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   writer[0]->compactVersion = INT64_MAX;
   writer[0]->now = taosGetTimestampMs();
 
-  code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr);
+  code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -1105,6 +1103,8 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) {
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
+    writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;
+
     taosThreadRwlockUnlock(&writer[0]->tsdb->rwLock);
   }
   tsdbFSEnableBgTask(tsdb->pFS);
@@ -1159,3 +1159,438 @@ _exit:
   }
   return code;
 }
+
+// snap part
+static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) {
+  if (x->minVer < y->minVer) return -1;
+  if (x->minVer > y->minVer) return 1;
+  if (x->maxVer < y->maxVer) return -1;
+  if (x->maxVer > y->maxVer) return 1;
+  return 0;
+}
+
+static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+STsdbSnapPartition* tsdbSnapPartitionCreate() {
+  STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition));
+  if (pSP == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
+    TARRAY2_INIT(&pSP->verRanges[i]);
+  }
+  return pSP;
+}
+
+void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) {
+  if (ppSP == NULL || ppSP[0] == NULL) {
+    return;
+  }
+  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
+    TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL);
+  }
+  taosMemoryFree(ppSP[0]);
+  ppSP[0] = NULL;
+}
+
+static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) {
+  switch (ftype) {
+    case TSDB_FTYPE_HEAD:
+      return TSDB_SNAP_RANGE_TYP_HEAD;
+    case TSDB_FTYPE_DATA:
+      return TSDB_SNAP_RANGE_TYP_DATA;
+    case TSDB_FTYPE_SMA:
+      return TSDB_SNAP_RANGE_TYP_SMA;
+    case TSDB_FTYPE_TOMB:
+      return TSDB_SNAP_RANGE_TYP_TOMB;
+    case TSDB_FTYPE_STT:
+      return TSDB_SNAP_RANGE_TYP_STT;
+  }
+  return TSDB_SNAP_RANGE_TYP_MAX;
+}
+
+static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) {
+  STsdbSnapPartition* p = tsdbSnapPartitionCreate();
+  if (p == NULL) {
+    goto _err;
+  }
+
+  p->fid = fset->fid;
+
+  int32_t code = 0;
+  int32_t typ = 0;
+  int32_t corrupt = false;
+  int32_t count = 0;
+  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+    if (fset->farr[ftype] == NULL) continue;
+    typ = tsdbFTypeToSRangeTyp(ftype);
+    ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX);
+    STFile*       f = fset->farr[ftype]->f;
+    if (f->maxVer > fset->maxVerValid) {
+      corrupt = true;
+      tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                ", ftype: %d",
+                fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype);
+      continue;
+    }
+    count++;
+    SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+
+  typ = TSDB_SNAP_RANGE_TYP_STT;
+  const SSttLvl* lvl;
+  TARRAY2_FOREACH(fset->lvlArr, lvl) {
+    STFileObj* fobj;
+    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+      STFile*       f = fobj->f;
+      if (f->maxVer > fset->maxVerValid) {
+        corrupt = true;
+        tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                  ", ftype: %d",
+                  fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ);
+        continue;
+      }
+      count++;
+      SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+      code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+      ASSERT(code == 0);
+    }
+  }
+  if (corrupt && count == 0) {
+    SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppSP[0] = p;
+  return 0;
+
+_err:
+  tsdbSnapPartitionClear(&p);
+  return -1;
+}
+
+STsdbSnapPartList* tsdbSnapPartListCreate() {
+  STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList));
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  TARRAY2_INIT(pList);
+  return pList;
+}
+
+static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) {
+  STsdbSnapPartList* pList = tsdbSnapPartListCreate();
+  if (pList == NULL) {
+    return NULL;
+  }
+
+  int32_t code = 0;
+  taosThreadRwlockRdlock(&fs->tsdb->rwLock);
+  STFileSet* fset;
+  TARRAY2_FOREACH(fs->fSetArr, fset) {
+    STsdbSnapPartition* pItem = NULL;
+    if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) {
+      code = -1;
+      break;
+    }
+    ASSERT(pItem != NULL);
+    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn);
+    ASSERT(code == 0);
+  }
+  taosThreadRwlockUnlock(&fs->tsdb->rwLock);
+
+  if (code) {
+    TARRAY2_DESTROY(pList, tsdbSnapPartitionClear);
+    taosMemoryFree(pList);
+    pList = NULL;
+  }
+  return pList;
+}
+
+int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) {
+  int32_t hdrLen = sizeof(int32_t);
+  int32_t datLen = 0;
+
+  int8_t  msgVer = 1;
+  int32_t len = TARRAY2_SIZE(pList);
+  hdrLen += sizeof(msgVer);
+  hdrLen += sizeof(len);
+  datLen += hdrLen;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
+    int32_t             typMax = TSDB_SNAP_RANGE_TYP_MAX;
+    int32_t             uItem = 0;
+    uItem += sizeof(STsdbSnapPartition);
+    uItem += sizeof(typMax);
+
+    for (int32_t i = 0; i < typMax; i++) {
+      int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]);
+      int32_t jItem = 0;
+      jItem += sizeof(SVersionRange);
+      jItem += sizeof(int64_t);
+      uItem += sizeof(iLen) + jItem * iLen;
+    }
+    datLen += uItem;
+  }
+  return datLen;
+}
+
+int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  int8_t  msgVer = 1;
+  int32_t len = TARRAY2_SIZE(pList);
+
+  if (tStartEncode(&encoder) < 0) goto _err;
+  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
+  if (tEncodeI32(&encoder, len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
+    if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
+    if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
+    if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
+    if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
+
+    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
+    if (tEncodeI32(&encoder, typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = TARRAY2_SIZE(iList);
+
+      if (tEncodeI32(&encoder, iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = TARRAY2_GET(iList, j);
+        if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
+      }
+    }
+  }
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+
+_err:
+  tEncoderClear(&encoder);
+  return -1;
+}
+
+int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  STsdbSnapPartition* p = NULL;
+
+  int8_t  msgVer = 0;
+  int32_t len = 0;
+  if (tStartDecode(&decoder) < 0) goto _err;
+  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
+  if (tDecodeI32(&decoder, &len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    p = tsdbSnapPartitionCreate();
+    if (p == NULL) goto _err;
+    if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
+    if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
+    if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
+    if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
+
+    int32_t typMax = 0;
+    if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = 0;
+      if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = {0};
+        if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
+        TARRAY2_APPEND(iList, r);
+      }
+    }
+    TARRAY2_APPEND(pList, p);
+    p = NULL;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+
+_err:
+  if (p) {
+    tsdbSnapPartitionClear(&p);
+  }
+  tDecoderClear(&decoder);
+  return -1;
+}
+
+int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) {
+  TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray));
+  if (pDiff == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  TARRAY2_INIT(pDiff);
+
+  STsdbSnapPartition* part;
+  TARRAY2_FOREACH(pList, part) {
+    STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange));
+    if (r == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+    int64_t maxVerValid = -1;
+    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &part->verRanges[i];
+      SVersionRange  vr = {0};
+      TARRAY2_FOREACH(iList, vr) {
+        if (vr.maxVer < vr.minVer) {
+          continue;
+        }
+        maxVerValid = TMAX(maxVerValid, vr.maxVer);
+      }
+    }
+    r->fid = part->fid;
+    r->sver = maxVerValid + 1;
+    r->ever = VERSION_MAX;
+    tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever);
+    int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppRanges[0] = pDiff;
+
+  tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff));
+  return 0;
+
+_err:
+  if (pDiff) {
+    tsdbSnapRangeArrayDestroy(&pDiff);
+  }
+  return -1;
+}
+
+void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) {
+  if (ppSnap && ppSnap[0]) {
+    TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear);
+    taosMemoryFree(ppSnap[0]);
+    ppSnap[0] = NULL;
+  }
+}
+
+void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) {
+  if (ppList == NULL || ppList[0] == NULL) return;
+
+  TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear);
+  taosMemoryFree(ppList[0]);
+  ppList[0] = NULL;
+}
+
+ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
+  if (!VND_IS_RSMA(pVnode)) {
+    return pVnode->pTsdb->pFS->fsstate;
+  }
+  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
+    if (SMA_RSMA_GET_TSDB(pVnode, lvl)->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
+      return TSDB_FS_STATE_INCOMPLETE;
+    }
+  }
+  return TSDB_FS_STATE_NORMAL;
+}
+
+int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) {
+  int                code = -1;
+  int32_t            tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX);
+  int32_t            subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
+  STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0};
+
+  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
+    pLists[j] = tsdbGetSnapPartList(pTsdb->pFS);
+    if (pLists[j] == NULL) goto _out;
+  }
+
+  // estimate bufLen and prepare
+  int32_t bufLen = sizeof(SSyncTLV);  // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY
+  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
+    bufLen += sizeof(SSyncTLV);  // subTyps[j]
+    bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]);
+  }
+
+  tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen);
+
+  void* data = taosMemoryRealloc(pSnap->data, bufLen);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen);
+    goto _out;
+  }
+  pSnap->data = data;
+
+  // header
+  SSyncTLV* head = data;
+  head->len = 0;
+  head->typ = pSnap->type;
+  int32_t offset = sizeof(SSyncTLV);
+  int32_t tlen = 0;
+
+  // fill snapshot info
+  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
+    if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    }
+
+    //  subHead
+    SSyncTLV* subHead = (void*)((char*)data + offset);
+    subHead->typ = subTyps[j];
+    ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV));
+
+    if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) {
+      tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr());
+      goto _out;
+    }
+    subHead->len = tlen;
+    offset += sizeof(SSyncTLV) + tlen;
+  }
+
+  head->len = offset - sizeof(SSyncTLV);
+  ASSERT(offset <= bufLen);
+  code = 0;
+
+_out:
+  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
+    if (pLists[j] == NULL) continue;
+    tsdbSnapPartListDestroy(&pLists[j]);
+  }
+
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbSttFileRW.c

@@ -14,6 +14,7 @@
  */
 
 #include "tsdbSttFileRW.h"
+#include "tsdbDataFileRW.h"
 
 // SSttFReader ============================================================
 struct SSttFileReader {
@@ -383,6 +384,8 @@ struct SSttFileWriter {
   struct {
     bool    opened;
     TABLEID tbid[1];
+    // range
+    SVersionRange range;
   } ctx[1];
   // file
   STsdbFD *fd;
@@ -401,8 +404,8 @@ struct SSttFileWriter {
   uint8_t *bufArr[5];
 };
 
-int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize,
+static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmprAlg, int64_t *fileSize,
-                                 TSttBlkArray *sttBlkArray, uint8_t **bufArr) {
+                                 TSttBlkArray *sttBlkArray, uint8_t **bufArr, SVersionRange *range) {
   if (blockData->nRow == 0) return 0;
 
   int32_t code = 0;
@@ -425,6 +428,8 @@ int32_t tsdbFileDoWriteBlockData(STsdbFD *fd, SBlockData *blockData, int8_t cmpr
     if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow];
   }
 
+  tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer);
+
   int32_t sizeArr[5] = {0};
   code = tCmprBlockData(blockData, cmprAlg, NULL, NULL, bufArr, sizeArr);
   if (code) return code;
@@ -455,8 +460,8 @@ static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
   int32_t code = 0;
   int32_t lino = 0;
 
-  code = tsdbFileDoWriteBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size,
+  code = tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, writer->config->cmprAlg, &writer->file->size,
-                                  writer->sttBlkArray, writer->config->bufArr);
+                                     writer->sttBlkArray, writer->config->bufArr, &writer->ctx->range);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -517,62 +522,6 @@ _exit:
   return code;
 }
 
-int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
-                               TTombBlkArray *tombBlkArray, uint8_t **bufArr) {
-  int32_t code;
-
-  if (TOMB_BLOCK_SIZE(tombBlock) == 0) return 0;
-
-  STombBlk tombBlk[1] = {{
-      .dp[0] =
-          {
-              .offset = *fileSize,
-              .size = 0,
-          },
-      .minTbid =
-          {
-              .suid = TARRAY2_FIRST(tombBlock->suid),
-              .uid = TARRAY2_FIRST(tombBlock->uid),
-          },
-      .maxTbid =
-          {
-              .suid = TARRAY2_LAST(tombBlock->suid),
-              .uid = TARRAY2_LAST(tombBlock->uid),
-          },
-      .minVer = TARRAY2_FIRST(tombBlock->version),
-      .maxVer = TARRAY2_FIRST(tombBlock->version),
-      .numRec = TOMB_BLOCK_SIZE(tombBlock),
-      .cmprAlg = cmprAlg,
-  }};
-
-  for (int32_t i = 1; i < TOMB_BLOCK_SIZE(tombBlock); i++) {
-    if (tombBlk->minVer > TARRAY2_GET(tombBlock->version, i)) {
-      tombBlk->minVer = TARRAY2_GET(tombBlock->version, i);
-    }
-    if (tombBlk->maxVer < TARRAY2_GET(tombBlock->version, i)) {
-      tombBlk->maxVer = TARRAY2_GET(tombBlock->version, i);
-    }
-  }
-
-  for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->dataArr); i++) {
-    code = tsdbCmprData((uint8_t *)TARRAY2_DATA(&tombBlock->dataArr[i]), TARRAY2_DATA_LEN(&tombBlock->dataArr[i]),
-                        TSDB_DATA_TYPE_BIGINT, tombBlk->cmprAlg, &bufArr[0], 0, &tombBlk->size[i], &bufArr[1]);
-    if (code) return code;
-
-    code = tsdbWriteFile(fd, *fileSize, bufArr[0], tombBlk->size[i]);
-    if (code) return code;
-
-    tombBlk->dp->size += tombBlk->size[i];
-    *fileSize += tombBlk->size[i];
-  }
-
-  code = TARRAY2_APPEND_PTR(tombBlkArray, tombBlk);
-  if (code) return code;
-
-  tTombBlockClear(tombBlock);
-  return 0;
-}
-
 static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
   if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) return 0;
 
@@ -580,7 +529,7 @@ static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
   int32_t lino = 0;
 
   code = tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
-                                writer->tombBlkArray, writer->config->bufArr);
+                                writer->tombBlkArray, writer->config->bufArr, &writer->ctx->range);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -639,21 +588,6 @@ _exit:
   return code;
 }
 
-int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize) {
-  ptr->size = TARRAY2_DATA_LEN(tombBlkArray);
-  if (ptr->size > 0) {
-    ptr->offset = *fileSize;
-
-    int32_t code = tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(tombBlkArray), ptr->size);
-    if (code) {
-      return code;
-    }
-
-    *fileSize += ptr->size;
-  }
-  return 0;
-}
-
 static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
   int32_t code = 0;
   int32_t lino = 0;
@@ -694,6 +628,8 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
       .fid = writer->config->fid,
       .cid = writer->config->cid,
       .size = 0,
+      .minVer = VERSION_MAX,
+      .maxVer = VERSION_MIN,
       .stt[0] =
           {
               .level = writer->config->level,
@@ -713,6 +649,9 @@ static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
   TSDB_CHECK_CODE(code, lino, _exit);
   writer->file->size += sizeof(hdr);
 
+  // range
+  writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
+
   writer->ctx->opened = true;
 
 _exit:
@@ -782,6 +721,7 @@ static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *o
       .fid = writer->config->fid,
       .nf = writer->file[0],
   };
+  tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
 
   code = TARRAY2_APPEND(opArray, op);
   TSDB_CHECK_CODE(code, lino, _exit);

source/dnode/vnode/src/tsdb/tsdbSttFileRW.h

@@ -71,6 +71,9 @@ int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *pBlockData
 int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record);
 bool    tsdbSttFileWriterIsOpened(SSttFileWriter *writer);
 
+int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize);
+int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize);
+
 struct SSttFileWriterConfig {
   STsdb    *tsdb;
   int32_t   maxRow;
@@ -90,4 +93,4 @@ struct SSttFileWriterConfig {
 }
 #endif
 
-#endif /*_TSDB_STT_FILE_RW_H*/
+#endif /*_TSDB_STT_FILE_RW_H*/

source/dnode/vnode/src/tsdb/tsdbUpgrade.c

@@ -16,24 +16,15 @@
 #include "tsdbUpgrade.h"
 
 // old
-extern void    tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t);
+#include "tsdb.h"
-extern int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData);
+// extern void    tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t);
 
 // new
-extern int32_t save_fs(const TFileSetArray *arr, const char *fname);
+#include "tsdbDataFileRW.h"
-extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
+#include "tsdbFS2.h"
-extern int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, int8_t cmprAlg, int64_t *fileSize,
+#include "tsdbSttFileRW.h"
-                                      TBrinBlkArray *brinBlkArray, uint8_t **bufArr);
+// extern int32_t save_fs(const TFileSetArray *arr, const char *fname);
-extern int32_t tsdbFileWriteBrinBlk(STsdbFD *fd, TBrinBlkArray *brinBlkArray, SFDataPtr *ptr, int64_t *fileSize);
+// extern int32_t current_fname(STsdb *pTsdb, char *fname, EFCurrentT ftype);
-extern int32_t tsdbFileWriteHeadFooter(STsdbFD *fd, int64_t *fileSize, const SHeadFooter *footer);
-extern int32_t tsdbSttLvlInit(int32_t level, SSttLvl **lvl);
-extern int32_t tsdbSttLvlClear(SSttLvl **lvl);
-extern int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize);
-extern int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize);
-extern int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAlg, int64_t *fileSize,
-                                      TTombBlkArray *tombBlkArray, uint8_t **bufArr);
-extern int32_t tsdbFileWriteTombBlk(STsdbFD *fd, const TTombBlkArray *tombBlkArray, SFDataPtr *ptr, int64_t *fileSize);
-extern int32_t tsdbFileWriteTombFooter(STsdbFD *fd, const STombFooter *footer, int64_t *fileSize);
 
 static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *reader, STFileSet *fset) {
   int32_t code = 0;
@@ -78,6 +69,8 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
         .fid = fset->fid,
         .cid = pDFileSet->pHeadF->commitID,
         .size = pDFileSet->pHeadF->size,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
 
     code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_HEAD]);
@@ -127,16 +120,18 @@ static int32_t tsdbUpgradeHead(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
         TSDB_CHECK_CODE(code, lino, _exit);
 
         if (BRIN_BLOCK_SIZE(ctx->brinBlock) >= ctx->maxRow) {
+          SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
           code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
-                                        ctx->brinBlkArray, ctx->bufArr);
+                                        ctx->brinBlkArray, ctx->bufArr, &range);
           TSDB_CHECK_CODE(code, lino, _exit);
         }
       }
     }
 
     if (BRIN_BLOCK_SIZE(ctx->brinBlock) > 0) {
+      SVersionRange range = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
       code = tsdbFileWriteBrinBlock(ctx->fd, ctx->brinBlock, ctx->cmprAlg, &fset->farr[TSDB_FTYPE_HEAD]->f->size,
-                                    ctx->brinBlkArray, ctx->bufArr);
+                                    ctx->brinBlkArray, ctx->bufArr, &range);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
 
@@ -182,6 +177,8 @@ static int32_t tsdbUpgradeData(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *
       .fid = fset->fid,
       .cid = pDFileSet->pDataF->commitID,
       .size = pDFileSet->pDataF->size,
+      .minVer = VERSION_MAX,
+      .maxVer = VERSION_MIN,
   };
 
   code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_DATA]);
@@ -208,6 +205,8 @@ static int32_t tsdbUpgradeSma(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReader *r
       .fid = fset->fid,
       .cid = pDFileSet->pSmaF->commitID,
       .size = pDFileSet->pSmaF->size,
+      .minVer = VERSION_MAX,
+      .maxVer = VERSION_MIN,
   };
 
   code = tsdbTFileObjInit(tsdb, &file, &fset->farr[TSDB_FTYPE_SMA]);
@@ -253,6 +252,8 @@ static int32_t tsdbUpgradeSttFile(STsdb *tsdb, SDFileSet *pDFileSet, SDataFReade
         .fid = fset->fid,
         .cid = pSttF->commitID,
         .size = pSttF->size,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
     code = tsdbTFileObjInit(tsdb, &file, &fobj);
     TSDB_CHECK_CODE(code, lino, _exit1);
@@ -382,6 +383,8 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
         .fid = fset->fid,
         .cid = 0,
         .size = 0,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
 
     code = tsdbTFileObjInit(tsdb, &file, fobj);
@@ -398,6 +401,8 @@ static int32_t tsdbUpgradeOpenTombFile(STsdb *tsdb, STFileSet *fset, STsdbFD **f
         .fid = fset->fid,
         .cid = 0,
         .size = 0,
+        .minVer = VERSION_MAX,
+        .maxVer = VERSION_MIN,
     };
 
     code = tsdbTFileObjInit(tsdb, &file, fobj);
@@ -481,8 +486,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
           code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt);
           TSDB_CHECK_CODE(code, lino, _exit);
         }
+        SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
         code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
-                                      ctx->bufArr);
+                                      ctx->bufArr, &tombRange);
         TSDB_CHECK_CODE(code, lino, _exit);
       }
     }
@@ -493,8 +499,9 @@ static int32_t tsdbDumpTombDataToFSet(STsdb *tsdb, SDelFReader *reader, SArray *
       code = tsdbUpgradeOpenTombFile(tsdb, fset, &ctx->fd, &ctx->fobj, &ctx->toStt);
       TSDB_CHECK_CODE(code, lino, _exit);
     }
+    SVersionRange tombRange = {.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
     code = tsdbFileWriteTombBlock(ctx->fd, ctx->tombBlock, ctx->cmprAlg, &ctx->fobj->f->size, ctx->tombBlkArray,
-                                  ctx->bufArr);
+                                  ctx->bufArr, &tombRange);
     TSDB_CHECK_CODE(code, lino, _exit);
   }
 

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -285,6 +285,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
   int32_t code = 0;
   int32_t lino = 0;
   char    dir[TSDB_FILENAME_LEN] = {0};
+  int64_t lastCommitted = pInfo->info.state.committed;
 
   tsem_wait(&pVnode->canCommit);
 

source/dnode/vnode/src/vnd/vnodeInitApi.c

@@ -131,6 +131,7 @@ void initTqAPI(SStoreTqReader* pTq) {
   pTq->tqGetResultBlock = tqGetResultBlock;
 
   pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
+  pTq->tqGetResultBlockTime = tqGetResultBlockTime;
 }
 
 void initStateStoreAPI(SStateStore* pStore) {

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "sync.h"
+#include "tsdb.h"
 #include "vnd.h"
 #include "vndCos.h"
 
@@ -62,6 +64,13 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, int32_t diskPrimary, STfs
   info.state.applied = -1;
   info.state.commitID = 0;
 
+  SVnodeInfo oldInfo = {0};
+  oldInfo.config = vnodeCfgDefault;
+  if (vnodeLoadInfo(dir, &oldInfo) == 0) {
+    vWarn("vgId:%d, vnode config info already exists at %s.", oldInfo.config.vgId, dir);
+    return (oldInfo.config.dbId == info.config.dbId) ? 0 : -1;
+  }
+
   vInfo("vgId:%d, save config while create", info.config.vgId);
   if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir) < 0) {
     vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno));
@@ -321,12 +330,13 @@ static int32_t vnodeCheckDisk(int32_t diskPrimary, STfs *pTfs) {
   return 0;
 }
 
-SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb) {
+SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force) {
   SVnode    *pVnode = NULL;
   SVnodeInfo info = {0};
   char       dir[TSDB_FILENAME_LEN] = {0};
   char       tdir[TSDB_FILENAME_LEN * 2] = {0};
   int32_t    ret = 0;
+  terrno = TSDB_CODE_SUCCESS;
 
   if (vnodeCheckDisk(diskPrimary, pTfs)) {
     vError("failed to open vnode from %s since %s. diskPrimary:%d", path, terrstr(), diskPrimary);
@@ -340,9 +350,14 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
   ret = vnodeLoadInfo(dir, &info);
   if (ret < 0) {
     vError("failed to open vnode from %s since %s", path, tstrerror(terrno));
+    terrno = TSDB_CODE_NEED_RETRY;
     return NULL;
   }
 
+  if (vnodeMkDir(pTfs, path)) {
+    vError("vgId:%d, failed to prepare vnode dir since %s, path: %s", info.config.vgId, strerror(errno), path);
+    return NULL;
+  }
   // save vnode info on dnode ep changed
   bool      updated = false;
   SSyncCfg *pCfg = &info.config.syncCfg;
@@ -404,7 +419,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
   }
 
   // open tsdb
-  if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback) < 0) {
+  if (!VND_IS_RSMA(pVnode) && tsdbOpen(pVnode, &VND_TSDB(pVnode), VNODE_TSDB_DIR, NULL, rollback, force) < 0) {
     vError("vgId:%d, failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
@@ -438,7 +453,7 @@ SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgC
   }
 
   // open sma
-  if (smaOpen(pVnode, rollback)) {
+  if (smaOpen(pVnode, rollback, force)) {
     vError("vgId:%d, failed to open vnode sma since %s", TD_VID(pVnode), tstrerror(terrno));
     goto _err;
   }
@@ -508,7 +523,10 @@ void vnodeClose(SVnode *pVnode) {
 }
 
 // start the sync timer after the queue is ready
-int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); }
+int32_t vnodeStart(SVnode *pVnode) {
+  ASSERT(pVnode);
+  return vnodeSyncStart(pVnode);
+}
 
 int32_t vnodeIsCatchUp(SVnode *pVnode) { return syncIsCatchUp(pVnode->sync); }
 
@@ -517,10 +535,3 @@ ESyncRole vnodeGetRole(SVnode *pVnode) { return syncGetRole(pVnode->sync); }
 void vnodeStop(SVnode *pVnode) {}
 
 int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
-
-void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) {
-  pSnapshot->data = NULL;
-  pSnapshot->lastApplyIndex = pVnode->state.committed;
-  pSnapshot->lastApplyTerm = pVnode->state.commitTerm;
-  pSnapshot->lastConfigIndex = -1;
-}

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -14,6 +14,7 @@
  */
 
 #include "vnd.h"
+#include "tsdb.h"
 
 // SVSnapReader ========================================================
 struct SVSnapReader {
@@ -28,6 +29,7 @@ struct SVSnapReader {
   SMetaSnapReader *pMetaReader;
   // tsdb
   int8_t           tsdbDone;
+  TSnapRangeArray *pRanges;
   STsdbSnapReader *pTsdbReader;
   // tq
   int8_t           tqHandleDone;
@@ -43,11 +45,84 @@ struct SVSnapReader {
   SStreamStateReader *pStreamStateReader;
   // rsma
   int8_t           rsmaDone;
+  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapReader *pRsmaReader;
 };
 
-int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) {
+static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) {
+  int32_t            code = -1;
+  STsdbSnapPartList *pList = tsdbSnapPartListCreate();
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _out;
+  }
+  if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) {
+    terrno = TSDB_CODE_INVALID_DATA_FMT;
+    goto _out;
+  }
+  if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) {
+    goto _out;
+  }
+  code = 0;
+_out:
+  tsdbSnapPartListDestroy(&pList);
+  return code;
+}
+
+static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
+  ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size");
+  switch (tsdbTyp) {
+    case SNAP_DATA_TSDB:
+      return &pReader->pRanges;
+    case SNAP_DATA_RSMA1:
+      return &pReader->pRsmaRanges[0];
+    case SNAP_DATA_RSMA2:
+      return &pReader->pRsmaRanges[1];
+    default:
+      return NULL;
+  }
+}
+
+static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
+  SVnode *pVnode = pReader->pVnode;
+  int32_t code = -1;
+
+  if (pParam->data) {
+    SSyncTLV *datHead = (void *)pParam->data;
+    if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+      terrno = TSDB_CODE_INVALID_DATA_FMT;
+      goto _out;
+    }
+
+    TSnapRangeArray **ppRanges = NULL;
+    int32_t           offset = 0;
+
+    while (offset + sizeof(SSyncTLV) < datHead->len) {
+      SSyncTLV *subField = (void *)(datHead->val + offset);
+      offset += sizeof(SSyncTLV) + subField->len;
+      void   *buf = subField->val;
+      int32_t bufLen = subField->len;
+      ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
+      if (ppRanges == NULL) {
+        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+        goto _out;
+      }
+      if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
+        vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
+        goto _out;
+      }
+    }
+  }
+
+  code = 0;
+_out:
+  return code;
+}
+
+int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader **ppReader) {
   int32_t       code = 0;
+  int64_t       sver = pParam->start;
+  int64_t       ever = pParam->end;
   SVSnapReader *pReader = NULL;
 
   pReader = (SVSnapReader *)taosMemoryCalloc(1, sizeof(*pReader));
@@ -59,6 +134,11 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe
   pReader->sver = sver;
   pReader->ever = ever;
 
+  // snapshot info
+  if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) {
+    goto _err;
+  }
+
   vInfo("vgId:%d, vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever);
   *ppReader = pReader;
   return code;
@@ -69,8 +149,19 @@ _err:
   return code;
 }
 
+static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) {
+  int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
+  for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
+    TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]);
+    if (ppRanges == NULL) continue;
+    tsdbSnapRangeArrayDestroy(ppRanges);
+  }
+}
+
 void vnodeSnapReaderClose(SVSnapReader *pReader) {
   vInfo("vgId:%d, close vnode snapshot reader", TD_VID(pReader->pVnode));
+  vnodeSnapReaderDestroyTsdbRanges(pReader);
+
   if (pReader->pRsmaReader) {
     rsmaSnapReaderClose(&pReader->pRsmaReader);
   }
@@ -175,7 +266,7 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
   if (!pReader->tsdbDone) {
     // open if not
     if (pReader->pTsdbReader == NULL) {
-      code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB,
+      code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, SNAP_DATA_TSDB, pReader->pRanges,
                                 &pReader->pTsdbReader);
       if (code) goto _err;
     }
@@ -364,6 +455,7 @@ struct SVSnapWriter {
   // meta
   SMetaSnapWriter *pMetaSnapWriter;
   // tsdb
+  TSnapRangeArray *pRanges;
   STsdbSnapWriter *pTsdbSnapWriter;
   // tq
   STqSnapWriter   *pTqSnapWriter;
@@ -373,12 +465,65 @@ struct SVSnapWriter {
   SStreamTaskWriter  *pStreamTaskWriter;
   SStreamStateWriter *pStreamStateWriter;
   // rsma
+  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapWriter *pRsmaSnapWriter;
 };
 
-int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) {
+TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
+  ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size");
+  switch (tsdbTyp) {
+    case SNAP_DATA_TSDB:
+      return &pWriter->pRanges;
+    case SNAP_DATA_RSMA1:
+      return &pWriter->pRsmaRanges[0];
+    case SNAP_DATA_RSMA2:
+      return &pWriter->pRsmaRanges[1];
+    default:
+      return NULL;
+  }
+}
+
+static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
+  SVnode *pVnode = pWriter->pVnode;
+  int32_t code = -1;
+
+  if (pParam->data) {
+    SSyncTLV *datHead = (void *)pParam->data;
+    if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+      terrno = TSDB_CODE_INVALID_DATA_FMT;
+      goto _out;
+    }
+
+    TSnapRangeArray **ppRanges = NULL;
+    int32_t           offset = 0;
+
+    while (offset + sizeof(SSyncTLV) < datHead->len) {
+      SSyncTLV *subField = (void *)(datHead->val + offset);
+      offset += sizeof(SSyncTLV) + subField->len;
+      void   *buf = subField->val;
+      int32_t bufLen = subField->len;
+      ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
+      if (ppRanges == NULL) {
+        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+        goto _out;
+      }
+      if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
+        vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
+        goto _out;
+      }
+    }
+  }
+
+  code = 0;
+_out:
+  return code;
+}
+
+int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
   SVSnapWriter *pWriter = NULL;
+  int64_t       sver = pParam->start;
+  int64_t       ever = pParam->end;
 
   // commit memory data
   vnodeAsyncCommit(pVnode);
@@ -397,6 +542,11 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr
   // inc commit ID
   pWriter->commitID = ++pVnode->state.commitID;
 
+  // snapshot info
+  if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) {
+    goto _err;
+  }
+
   vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode),
         sver, ever, pWriter->commitID);
   *ppWriter = pWriter;
@@ -408,15 +558,30 @@ _err:
   return code;
 }
 
+static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) {
+  int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
+  for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
+    TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]);
+    if (ppRanges == NULL) continue;
+    tsdbSnapRangeArrayDestroy(ppRanges);
+  }
+}
+
 int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot) {
   int32_t code = 0;
   SVnode *pVnode = pWriter->pVnode;
 
+  vnodeSnapWriterDestroyTsdbRanges(pWriter);
+
   // prepare
   if (pWriter->pTsdbSnapWriter) {
     tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter);
   }
 
+  if (pWriter->pRsmaSnapWriter) {
+    rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter);
+  }
+
   // commit json
   if (!rollback) {
     pWriter->info.state.committed = pWriter->ever;
@@ -430,7 +595,9 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     char dir[TSDB_FILENAME_LEN] = {0};
     vnodeGetPrimaryDir(pVnode->path, pVnode->diskPrimary, pVnode->pTfs, dir, TSDB_FILENAME_LEN);
 
-    vnodeCommitInfo(dir);
+    code = vnodeCommitInfo(dir);
+    if (code) goto _exit;
+
   } else {
     vnodeRollback(pWriter->pVnode);
   }
@@ -561,7 +728,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
     case SNAP_DATA_DEL: {
       // tsdb
       if (pWriter->pTsdbSnapWriter == NULL) {
-        code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter);
+        code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, pWriter->pRanges,
+                                  &pWriter->pTsdbSnapWriter);
         if (code) goto _err;
       }
 
@@ -621,7 +789,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
     case SNAP_DATA_QTASK: {
       // rsma1/rsma2/qtask for rsma
       if (pWriter->pRsmaSnapWriter == NULL) {
-        code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, &pWriter->pRsmaSnapWriter);
+        code = rsmaSnapWriterOpen(pVnode->pSma, pWriter->sver, pWriter->ever, (void **)pWriter->pRsmaRanges,
+                                  &pWriter->pRsmaSnapWriter);
         if (code) goto _err;
       }
 

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -15,6 +15,8 @@
 
 #define _DEFAULT_SOURCE
 #include "tq.h"
+#include "sync.h"
+#include "tsdb.h"
 #include "vnd.h"
 
 #define BATCH_ENABLE 0
@@ -416,8 +418,8 @@ static int32_t vnodeSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
   return code;
 }
 
-static void vnodeSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+static int32_t vnodeSyncGetSnapshotInfo(const SSyncFSM *pFsm, SSnapshot *pSnapshot) {
-  vnodeGetSnapshot(pFsm->data, pSnapshot);
+  return vnodeGetSnapshot(pFsm->data, pSnapshot);
 }
 
 static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) {
@@ -475,8 +477,7 @@ static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, SFsmCbMeta
 
 static int32_t vnodeSnapshotStartRead(const SSyncFSM *pFsm, void *pParam, void **ppReader) {
   SVnode         *pVnode = pFsm->data;
-  SSnapshotParam *pSnapshotParam = pParam;
+  int32_t         code = vnodeSnapReaderOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapReader **)ppReader);
-  int32_t code = vnodeSnapReaderOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapReader **)ppReader);
   return code;
 }
 
@@ -492,8 +493,7 @@ static int32_t vnodeSnapshotDoRead(const SSyncFSM *pFsm, void *pReader, void **p
 }
 
 static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void **ppWriter) {
-  SVnode         *pVnode = pFsm->data;
+  SVnode *pVnode = pFsm->data;
-  SSnapshotParam *pSnapshotParam = pParam;
 
   do {
     int32_t itemSize = tmsgGetQueueSize(&pVnode->msgCb, pVnode->config.vgId, APPLY_QUEUE);
@@ -506,7 +506,7 @@ static int32_t vnodeSnapshotStartWrite(const SSyncFSM *pFsm, void *pParam, void
     }
   } while (true);
 
-  int32_t code = vnodeSnapWriterOpen(pVnode, pSnapshotParam->start, pSnapshotParam->end, (SVSnapWriter **)ppWriter);
+  int32_t code = vnodeSnapWriterOpen(pVnode, (SSnapshotParam *)pParam, (SVSnapWriter **)ppWriter);
   return code;
 }
 
@@ -642,6 +642,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) {
   pFsm->FpAppliedIndexCb = vnodeSyncAppliedIndex;
   pFsm->FpPreCommitCb = vnodeSyncPreCommitMsg;
   pFsm->FpRollBackCb = vnodeSyncRollBackMsg;
+  pFsm->FpGetSnapshot = NULL;
   pFsm->FpGetSnapshotInfo = vnodeSyncGetSnapshotInfo;
   pFsm->FpRestoreFinishCb = vnodeRestoreFinish;
   pFsm->FpLeaderTransferCb = NULL;
@@ -784,3 +785,20 @@ bool vnodeIsLeader(SVnode *pVnode) {
 
   return true;
 }
+
+int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
+  int code = 0;
+  pSnap->lastApplyIndex = pVnode->state.committed;
+  pSnap->lastApplyTerm = pVnode->state.commitTerm;
+  pSnap->lastConfigIndex = -1;
+  pSnap->state = SYNC_FSM_STATE_COMPLETE;
+
+  if (tsdbSnapGetFsState(pVnode) != TSDB_FS_STATE_NORMAL) {
+    pSnap->state = SYNC_FSM_STATE_INCOMPLETE;
+  }
+
+  if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    code = tsdbSnapGetDetails(pVnode, pSnap);
+  }
+  return code;
+}

source/libs/executor/inc/executorInt.h

@@ -636,6 +636,7 @@ typedef struct SStreamFillSupporter {
   SSHashObj*     pResMap;
   bool           hasDelete;
   SStorageAPI*   pAPI;
+  STimeWindow    winRange;
 } SStreamFillSupporter;
 
 typedef struct SStreamFillOperatorInfo {
@@ -770,6 +771,7 @@ SSDataBlock* getNextBlockFromDownstreamImpl(struct SOperatorInfo* pOperator, int
 bool inSlidingWindow(SInterval* pInterval, STimeWindow* pWin, SDataBlockInfo* pBlockInfo);
 bool inCalSlidingWindow(SInterval* pInterval, STimeWindow* pWin, TSKEY calStart, TSKEY calEnd, EStreamType blockType);
 bool compareVal(const char* v, const SStateKeys* pKey);
+bool inWinRange(STimeWindow* range, STimeWindow* cur);
 
 int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
                                TSKEY* primaryKeys, int32_t prevPosition, int32_t order);

source/libs/executor/src/filloperator.c

@@ -950,7 +950,10 @@ static bool hasRemainCalc(SStreamFillInfo* pFillInfo) {
 
 static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
   while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
-    buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock);
+    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
+    if (inWinRange(&pFillSup->winRange, &st)) {
+      buildFillResult(pFillInfo->pResRow, pFillSup, pFillInfo->current, pBlock);
+    }
     pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
                                      pFillSup->interval.precision);
   }
@@ -960,7 +963,8 @@ static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo*
   while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
     uint64_t groupId = pBlock->info.id.groupId;
     SWinKey  key = {.groupId = groupId, .ts = pFillInfo->current};
-    if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) {
+    STimeWindow st = {.skey = pFillInfo->current, .ekey = pFillInfo->current};
+    if ( ( pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId) ) || !inWinRange(&pFillSup->winRange, &st) ) {
       pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
                                        pFillSup->interval.precision);
       pFillInfo->pLinearInfo->winIndex++;
@@ -1345,6 +1349,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
         pInfo->pFillInfo->preRowKey = INT64_MIN;
       }
 
+      pInfo->pFillSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;
+      if (pInfo->pFillSup->winRange.ekey <= 0) {
+        pInfo->pFillSup->winRange.ekey = INT64_MAX;
+      }
+
       switch (pBlock->info.type) {
         case STREAM_RETRIEVE:
           return pBlock;

source/libs/executor/src/projectoperator.c

@@ -110,7 +110,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
   pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder;
   pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder;
 
-  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
+  if (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM || pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {
     pInfo->mergeDataBlocks = false;
   } else {
     if (!pProjPhyNode->ignoreGroupId) {

source/libs/executor/src/scanoperator.c

@@ -1863,6 +1863,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
       // curVersion move to next
       tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion);
 
+      // use ts to pass time when replay, because ts not used if type is log
+      pTaskInfo->streamInfo.currentOffset.ts = pAPI->tqReaderFn.tqGetResultBlockTime(pInfo->tqReader);
+
       if (hasResult) {
         qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows,
                pTaskInfo->streamInfo.currentOffset.version);

source/libs/executor/src/streamtimewindowoperator.c

@@ -3605,7 +3605,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
   for (int32_t i = 0; i < num; i++) {
     SStateWindowInfo curInfo = {0};
     SStateWindowInfo nextInfo = {0};
-    SStateWindowInfo dummy = {0};
     qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,
            pSeKeyBuf[i].groupId, i);
     getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);

source/libs/nodes/src/nodesUtilFuncs.c

@@ -2033,7 +2033,6 @@ typedef struct SCollectFuncsCxt {
   char*           tableAlias;
   FFuncClassifier classifier;
   SNodeList*      pFuncs;
-  SHashObj*       pFuncsSet;
 } SCollectFuncsCxt;
 
 static EDealRes collectFuncs(SNode* pNode, void* pContext) {
@@ -2048,9 +2047,15 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) {
       }
     }
     SExprNode* pExpr = (SExprNode*)pNode;
-    if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, sizeof(SExprNode*))) {
+    bool bFound = false;
+    SNode* pn = NULL;
+    FOREACH(pn, pCxt->pFuncs) {
+      if (nodesEqualNode(pn, pNode)) {
+        bFound = true;
+      }
+    }
+    if (!bFound) {
       pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode));
-      taosHashPut(pCxt->pFuncsSet, &pExpr, POINTER_BYTES, &pExpr, POINTER_BYTES);
     }
     return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR);
   }
@@ -2077,12 +2082,10 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl
   SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS,
                           .classifier = classifier,
                           .tableAlias = tableAlias,
-                          .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs),
+                          .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs)};
-                          .pFuncsSet = taosHashInit(4, funcNodeHash, false, false)};
+  if (NULL == cxt.pFuncs) {
-  if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) {
     return TSDB_CODE_OUT_OF_MEMORY;
   }
-  taosHashSetEqualFp(cxt.pFuncsSet, funcNodeEqual);
   *pFuncs = NULL;
   nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt);
   if (TSDB_CODE_SUCCESS == cxt.errCode) {
@@ -2094,7 +2097,6 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, char* tableAl
   } else {
     nodesDestroyList(cxt.pFuncs);
   }
-  taosHashCleanup(cxt.pFuncsSet);
 
   return cxt.errCode;
 }