other: merge 3.0

Jenkinsfile2

@@ -393,7 +393,7 @@ pipeline {
                     agent{label " Mac_catalina "}
                     steps {
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 20, unit: 'MINUTES'){
+                            timeout(time: 30, unit: 'MINUTES'){
                                 pre_test()
                                 pre_test_build_mac()
                             }

docs/en/12-taos-sql/02-database.md

@@ -26,7 +26,6 @@ database_option: {
   | PAGESIZE  value
   | PRECISION {'ms' | 'us' | 'ns'}
   | REPLICA value
-  | RETENTIONS ingestion_duration:keep_duration ...
   | WAL_LEVEL {1 | 2}
   | VGROUPS value
   | SINGLE_STABLE {0 | 1}
@@ -61,7 +60,6 @@ database_option: {
 - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384.
 - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms.
 - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster.
-- RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods.
 - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1.
   - 1: WAL is enabled but fsync is disabled.
   - 2: WAL and fsync are both enabled.

docs/en/12-taos-sql/03-table.md

@@ -32,9 +32,6 @@ table_options:
 
 table_option: {
     COMMENT 'string_value'
-  | WATERMARK duration[,duration]
-  | MAX_DELAY duration[,duration]
-  | ROLLUP(func_name [, func_name] ...)
   | SMA(col_name [, col_name] ...)
   | TTL value
 }
@@ -54,11 +51,8 @@ table_option: {
 
 **Parameter description**
 1. COMMENT: specifies comments for the table. This parameter can be used with supertables, standard tables, and subtables.
-2. WATERMARK: specifies the time after which the window is closed. The default value is 5 seconds. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database.
+2. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables.
-3. MAX_DELAY: specifies the maximum latency for pushing computation results. The default value is 15 minutes or the value of the INTERVAL parameter, whichever is smaller. Enter a value between 0 and 15 minutes in milliseconds, seconds, or minutes. You can enter multiple values separated by commas (,). Note: Retain the default value if possible. Configuring a small MAX_DELAY may cause results to be frequently pushed, affecting storage and query performance. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database.
+3. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire.
-4. ROLLUP: specifies aggregate functions to roll up. Rolling up a function provides downsampled results based on multiple axes. This parameter applies only to supertables and takes effect only when the RETENTIONS parameter has been specified for the database. You can specify only one function to roll up. The rollup takes effect on all columns except TS. Enter one of the following values: avg, sum, min, max, last, or first.
-5. SMA: specifies functions on which to enable small materialized aggregates (SMA). SMA is user-defined precomputation of aggregates based on data blocks. Enter one of the following values: max, min, or sum This parameter can be used with supertables and standard tables.
-6. TTL: specifies the time to live (TTL) for the table. If TTL is specified when creatinga table, after the time period for which the table has been existing is over TTL, TDengine will automatically delete the table. Please be noted that the system may not delete the table at the exact moment that the TTL expires but guarantee there is such a system and finally the table will be deleted. The unit of TTL is in days. The default value is 0, i.e. never expire.
 
 ## Create Subtables
 

docs/examples/python/bind_param_example.py

@@ -20,7 +20,7 @@ def get_ts(ts: str):
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/conn_websocket_pandas.py

@@ -4,7 +4,7 @@ import taos
 
 taos_conn = taos.connect()
 taos_conn.execute('drop database if exists power')
-taos_conn.execute('create database if not exists power wal_retention_period 3600')
+taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ')
 taos_conn.execute("use power")
 taos_conn.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")

docs/examples/python/connect_rest_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power")
-cursor.execute("CREATE DATABASE power")
+cursor.execute("CREATE DATABASE power keep 36500 ")
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")
 

docs/examples/python/connect_rest_with_req_id_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power", req_id=1)
-cursor.execute("CREATE DATABASE power", req_id=2)
+cursor.execute("CREATE DATABASE power keep 36500", req_id=2)
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3)
 

docs/examples/python/connect_websocket_examples.py

@@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy")
-conn.execute("create database if not exists connwspy wal_retention_period 3600")
+conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ")
 conn.execute("use connwspy")
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)")
 conn.execute("create table if not exists tb1 using stb tags (1)")
 conn.execute("insert into tb1 values (now, 1)")
-conn.execute("insert into tb1 values (now, 2)")
+conn.execute("insert into tb1 values (now+1s, 2)")
-conn.execute("insert into tb1 values (now, 3)")
+conn.execute("insert into tb1 values (now+2s, 3)")
 
 r = conn.execute("select * from stb")
 result = conn.query("select * from stb")

docs/examples/python/connect_websocket_with_req_id_examples.py

@@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy", req_id=1)
-conn.execute("create database if not exists connwspy", req_id=2)
+conn.execute("create database if not exists connwspy keep 36500", req_id=2)
 conn.execute("use connwspy", req_id=3)
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4)
 conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5)

docs/examples/python/connection_usage_native_reference.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")

docs/examples/python/connection_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)

docs/examples/python/cursor_usage_native_reference.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test")
-cursor.execute("CREATE DATABASE test")
+cursor.execute("CREATE DATABASE test keep 36500")
 cursor.execute("USE test")
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 

docs/examples/python/cursor_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test", req_id=1)
-cursor.execute("CREATE DATABASE test", req_id=2)
+cursor.execute("CREATE DATABASE test keep 36500", req_id=2)
 cursor.execute("USE test", req_id=3)
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4)
 

docs/examples/python/fast_write_example.py

@@ -160,7 +160,7 @@ def main(infinity):
 
     conn = get_connection()
     conn.execute("DROP DATABASE IF EXISTS test")
-    conn.execute("CREATE DATABASE IF NOT EXISTS test")
+    conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500")
     conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")
     conn.close()

docs/examples/python/json_protocol_example.py

@@ -16,7 +16,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/kafka_example_common.py

@@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD
              'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
              'California.SantaClara', 'California.Cupertino']
 
-CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
+CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
 USE_DATABASE_SQL = 'use {}'
 DROP_TABLE_SQL = 'drop table if exists meters'
 DROP_DATABASE_SQL = 'drop database if exists {}'

docs/examples/python/line_protocol_example.py

@@ -15,7 +15,7 @@ def get_connection():
 
 def create_database(conn):
     # the default precision is ms (microsecond), but we use us(microsecond) here.
-    conn.execute("CREATE DATABASE test precision 'us'")
+    conn.execute("CREATE DATABASE test precision 'us' keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/multi_bind_example.py

@@ -71,7 +71,7 @@ def insert_data():
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/native_insert_example.py

@@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection:
 
 
 def create_stable(conn: taos.TaosConnection):
-    conn.execute("CREATE DATABASE power")
+    conn.execute("CREATE DATABASE power keep 36500")
     conn.execute("USE power")
     conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")

docs/examples/python/result_set_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 # prepare data

docs/examples/python/result_set_with_req_id_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)
 # prepare data

docs/examples/python/schemaless_insert.py

@@ -3,7 +3,7 @@ import taos
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_raw.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_req_id.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_ttl.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_req_id.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_ttl.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/sql_writer.py

@@ -10,7 +10,7 @@ class SQLWriter:
         self._tb_tags = {}
         self._conn = get_connection_func()
         self._max_sql_length = self.get_max_sql_length()
-        self._conn.execute("create database if not exists test")
+        self._conn.execute("create database if not exists test keep 36500")
         self._conn.execute("USE test")
 
     def get_max_sql_length(self):

docs/examples/python/stmt_example.py

@@ -10,7 +10,7 @@ db_name = 'test_ws_stmt'
 def before():
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/stmt_websocket_example.py

@@ -9,7 +9,7 @@ import taos
 def before_test(db_name):
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/telnet_line_protocol_example.py

@@ -19,7 +19,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/tmq_assignment_example.py

@@ -7,7 +7,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

docs/examples/python/tmq_example.py

@@ -6,7 +6,7 @@ def init_tmq_env(db, topic):
     conn = taos.connect()
     conn.execute("drop topic if exists {}".format(topic))
     conn.execute("drop database if exists {}".format(db))
-    conn.execute("create database if not exists {} wal_retention_period 3600".format(db))
+    conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db))
     conn.select_db(db)
     conn.execute(
         "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))")

docs/examples/python/tmq_websocket_assgnment_example.py

@@ -6,7 +6,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

docs/zh/12-taos-sql/02-database.md

@@ -26,7 +26,6 @@ database_option: {
   | PAGESIZE  value
   | PRECISION {'ms' | 'us' | 'ns'}
   | REPLICA value
-  | RETENTIONS ingestion_duration:keep_duration ...
   | WAL_LEVEL {1 | 2}
   | VGROUPS value
   | SINGLE_STABLE {0 | 1}
@@ -61,7 +60,6 @@ database_option: {
 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。
 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。
 - REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。
-- RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。
 - WAL_LEVEL：WAL 级别，默认为 1。
   - 1：写 WAL，但不执行 fsync。
   - 2：写 WAL，而且执行 fsync。

docs/zh/12-taos-sql/03-table.md

@@ -30,9 +30,6 @@ table_options:
 
 table_option: {
     COMMENT 'string_value'
-  | WATERMARK duration[,duration]
-  | MAX_DELAY duration[,duration]
-  | ROLLUP(func_name [, func_name] ...)
   | SMA(col_name [, col_name] ...)
   | TTL value
 }
@@ -52,11 +49,8 @@ table_option: {
 **参数说明**
 
 1. COMMENT：表注释。可用于超级表、子表和普通表。
-2. WATERMARK：指定窗口的关闭时间，默认值为 5 秒，最小单位毫秒，范围为 0 到 15 分钟，多个以逗号分隔。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。
+2. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。
-3. MAX_DELAY：用于控制推送计算结果的最大延迟，默认值为 interval 的值(但不能超过最大值)，最小单位毫秒，范围为 1 毫秒到 15 分钟，多个以逗号分隔。注：不建议 MAX_DELAY 设置太小，否则会过于频繁的推送结果，影响存储和查询性能，如无特殊需求，取默认值即可。只可用于超级表，且只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。
+3. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
-4. ROLLUP：Rollup 指定的聚合函数，提供基于多层级的降采样聚合结果。只可用于超级表。只有当数据库使用了 RETENTIONS 参数时，才可以使用此表参数。作用于超级表除 TS 列外的其它所有列，但是只能定义一个聚合函数。 聚合函数支持 avg, sum, min, max, last, first。
-5. SMA：Small Materialized Aggregates，提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。
-6. TTL：Time to Live，是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数，当该表的存在时间超过 TTL 指定的时间后，TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间，系统不保证到了时间一定会将其删除，而只保证存在这样一个机制且最终一定会删除。TTL 单位是天，默认为 0，表示不限制，到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联，如果 KEEP 比 TTL 小，在表被删除之前数据也可能已经被删除。
 
 ## 创建子表
 

include/common/tgrant.h

@@ -31,8 +31,6 @@ extern "C" {
 #endif
 
 #define GRANT_HEART_BEAT_MIN 2
-#define GRANT_ACTIVE_CODE    "activeCode"
-#define GRANT_C_ACTIVE_CODE  "cActiveCode"
 
 typedef enum {
   TSDB_GRANT_ALL,
@@ -52,11 +50,6 @@ typedef enum {
   TSDB_GRANT_TABLE,
 } EGrantType;
 
-typedef struct {
-  int64_t grantedTime;
-  int64_t connGrantedTime;
-} SGrantedInfo;
-
 int32_t grantCheck(EGrantType grant);
 int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
 

include/common/tmsg.h

@@ -35,12 +35,14 @@ extern "C" {
 #define TD_MSG_NUMBER_
 #undef TD_MSG_DICT_
 #undef TD_MSG_INFO_
+#undef TD_MSG_RANGE_CODE_
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
 #undef TD_MSG_NUMBER_
 #undef TD_MSG_DICT_
 #undef TD_MSG_INFO_
+#undef TD_MSG_RANGE_CODE_
 #define TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
@@ -48,33 +50,31 @@ extern "C" {
 #undef TD_MSG_DICT_
 #undef TD_MSG_INFO_
 #undef TD_MSG_SEG_CODE_
+#undef TD_MSG_RANGE_CODE_
 #include "tmsgdef.h"
 
 extern char*   tMsgInfo[];
 extern int32_t tMsgDict[];
-
+extern int32_t tMsgRangeDict[];
-#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
-#define TMSG_SEG_SEQ(TYPE)  ((TYPE)&0xff)
-#define TMSG_INFO(TYPE)                                                                                                \
-  ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \
-   (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) ||                         \
-          (TYPE) < TDMT_VND_STREAM_MSG || (TYPE) < TDMT_VND_TMQ_MSG || (TYPE) < TDMT_VND_TMQ_MAX_MSG                   \
-      ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)]                                                   \
-      : 0
-
-#define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
 
 typedef uint16_t tmsg_t;
 
+#define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8)
+#define TMSG_SEG_SEQ(TYPE)  ((TYPE)&0xff)
+#define TMSG_INDEX(TYPE)    (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
+
 static inline bool tmsgIsValid(tmsg_t type) {
-  if (type < TDMT_DND_MAX_MSG || type < TDMT_MND_MAX_MSG || type < TDMT_VND_MAX_MSG || type < TDMT_SCH_MAX_MSG ||
+  // static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]);
-      type < TDMT_STREAM_MAX_MSG || type < TDMT_MON_MAX_MSG || type < TDMT_SYNC_MAX_MSG || type < TDMT_VND_STREAM_MSG ||
+  int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG);
-      type < TDMT_VND_TMQ_MSG || type < TDMT_VND_TMQ_MAX_MSG) {
+  int    segIdx = TMSG_SEG_CODE(type);
-    return true;
+  if (segIdx >= 0 && segIdx < maxSegIdx) {
-  } else {
+    return type < tMsgRangeDict[segIdx];
-    return false;
   }
+  return false;
 }
+
+#define TMSG_INFO(type) (tmsgIsValid(type) ? tMsgInfo[TMSG_INDEX(type)] : "unKnown")
+
 static inline bool vnodeIsMsgBlock(tmsg_t type) {
   return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) ||
          (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT) ||
@@ -295,7 +295,36 @@ typedef enum ENodeType {
   QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_GRANT_STMT,
   QUERY_NODE_REVOKE_STMT,
-  QUERY_NODE_SHOW_DNODES_STMT,
+  // placeholder for [152, 180]
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
+  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_DELETE_STMT,
+  QUERY_NODE_INSERT_STMT,
+  QUERY_NODE_QUERY,
+  QUERY_NODE_SHOW_DB_ALIVE_STMT,
+  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_RESTORE_DNODE_STMT,
+  QUERY_NODE_RESTORE_QNODE_STMT,
+  QUERY_NODE_RESTORE_MNODE_STMT,
+  QUERY_NODE_RESTORE_VNODE_STMT,
+  QUERY_NODE_PAUSE_STREAM_STMT,
+  QUERY_NODE_RESUME_STREAM_STMT,
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
+
+  // show statement nodes
+  // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
+  QUERY_NODE_SHOW_DNODES_STMT = 400,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
@@ -324,31 +353,6 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_VIEWS_STMT,
-  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
-  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
-  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
-  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
-  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
-  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
-  QUERY_NODE_SHOW_SCORES_STMT,
-  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
-  QUERY_NODE_KILL_CONNECTION_STMT,
-  QUERY_NODE_KILL_QUERY_STMT,
-  QUERY_NODE_KILL_TRANSACTION_STMT,
-  QUERY_NODE_DELETE_STMT,
-  QUERY_NODE_INSERT_STMT,
-  QUERY_NODE_QUERY,
-  QUERY_NODE_SHOW_DB_ALIVE_STMT,
-  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
-  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
-  QUERY_NODE_RESTORE_DNODE_STMT,
-  QUERY_NODE_RESTORE_QNODE_STMT,
-  QUERY_NODE_RESTORE_MNODE_STMT,
-  QUERY_NODE_RESTORE_VNODE_STMT,
-  QUERY_NODE_PAUSE_STREAM_STMT,
-  QUERY_NODE_RESUME_STREAM_STMT,  
-  QUERY_NODE_CREATE_VIEW_STMT,
-  QUERY_NODE_DROP_VIEW_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@@ -790,7 +794,7 @@ typedef struct {
 
 int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
 int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq);
-void    tFreeSMDropStbReq(SMDropStbReq *pReq);
+void    tFreeSMDropStbReq(SMDropStbReq* pReq);
 
 typedef struct {
   char    name[TSDB_TABLE_FNAME_LEN];
@@ -878,9 +882,9 @@ typedef struct {
 
 int32_t tSerializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq);
 int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq);
-void    tFreeSDropUserReq(SDropUserReq *pReq);
+void    tFreeSDropUserReq(SDropUserReq* pReq);
 
-typedef struct SIpV4Range{
+typedef struct SIpV4Range {
   uint32_t ip;
   uint32_t mask;
 } SIpV4Range;
@@ -906,7 +910,7 @@ typedef struct {
 
 int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
 int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
-void    tFreeSCreateUserReq(SCreateUserReq *pReq);
+void    tFreeSCreateUserReq(SCreateUserReq* pReq);
 
 typedef struct {
   int64_t     ver;
@@ -1378,7 +1382,7 @@ typedef struct {
 
 int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
 int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
-void    tFreeSCompactDbReq(SCompactDbReq *pReq);
+void    tFreeSCompactDbReq(SCompactDbReq* pReq);
 
 typedef struct {
   char    name[TSDB_FUNC_NAME_LEN];
@@ -1817,7 +1821,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
 int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
 void    tFreeSViewHbRsp(SViewHbRsp* pRsp);
 
-
 typedef struct {
   int32_t numOfTables;
   int32_t numOfVgroup;
@@ -2006,7 +2009,7 @@ typedef struct {
 
 int32_t tSerializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq);
 int32_t tDeserializeSRestoreDnodeReq(void* buf, int32_t bufLen, SRestoreDnodeReq* pReq);
-void    tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq);
+void    tFreeSRestoreDnodeReq(SRestoreDnodeReq* pReq);
 
 typedef struct {
   int32_t dnodeId;
@@ -2018,7 +2021,7 @@ typedef struct {
 
 int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
 int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq);
-void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq);
+void    tFreeSMCfgDnodeReq(SMCfgDnodeReq* pReq);
 
 typedef struct {
   char config[TSDB_DNODE_CONFIG_LEN];
@@ -2037,7 +2040,7 @@ typedef struct {
 
 int32_t tSerializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq);
 int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq);
-void    tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq);
+void    tFreeSMCreateQnodeReq(SMCreateQnodeReq* pReq);
 void    tFreeSDDropQnodeReq(SDDropQnodeReq* pReq);
 typedef struct {
   int8_t   replica;
@@ -2079,7 +2082,7 @@ typedef struct {
 
 int32_t tSerializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq);
 int32_t tDeserializeSBalanceVgroupReq(void* buf, int32_t bufLen, SBalanceVgroupReq* pReq);
-void    tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq);
+void    tFreeSBalanceVgroupReq(SBalanceVgroupReq* pReq);
 
 typedef struct {
   int32_t vgId1;
@@ -2100,7 +2103,7 @@ typedef struct {
 
 int32_t tSerializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq);
 int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistributeVgroupReq* pReq);
-void    tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq);
+void    tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq);
 
 typedef struct {
   int32_t useless;
@@ -2111,7 +2114,7 @@ typedef struct {
 
 int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);
 int32_t tDeserializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);
-void    tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq);
+void    tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq* pReq);
 
 typedef struct {
   int32_t vgId;
@@ -2511,7 +2514,7 @@ typedef struct {
 
 int32_t tSerializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
 int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq);
-void    tFreeSMDropTopicReq(SMDropTopicReq *pReq);
+void    tFreeSMDropTopicReq(SMDropTopicReq* pReq);
 
 typedef struct {
   char   topic[TSDB_TOPIC_FNAME_LEN];
@@ -3942,7 +3945,6 @@ int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pR
 int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
 void    tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
 
-
 #pragma pack(pop)
 
 #ifdef __cplusplus

include/common/tmsgdef.h

@@ -24,48 +24,70 @@
 
 #if defined(TD_MSG_INFO_)
 
-#undef TD_NEW_MSG_SEG
+  #undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
+  #undef TD_DEF_MSG_TYPE
-#define TD_NEW_MSG_SEG(TYPE) "null",
+  #undef TD_CLOSE_MSG_TYPE
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp",
+  #define TD_NEW_MSG_SEG(TYPE) "null",
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp",
+  #define TD_CLOSE_MSG_TYPE(TYPE)
 
-char *tMsgInfo[] = {
+  char *tMsgInfo[] = {
+
+#elif defined(TD_MSG_RANGE_CODE_)
+
+  #undef TD_NEW_MSG_SEG
+  #undef TD_DEF_MSG_TYPE
+  #undef TD_CLOSE_MSG_TYPE
+  #define TD_NEW_MSG_SEG(TYPE) 
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
+  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  int32_t tMsgRangeDict[] = {
 
 #elif defined(TD_MSG_NUMBER_)
 
-#undef TD_NEW_MSG_SEG
+  #undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
+  #undef TD_DEF_MSG_TYPE
-#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
+  #undef TD_CLOSE_MSG_TYPE
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
+  #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
+  #define TD_CLOSE_MSG_TYPE(TYPE)
 
-enum {
+  enum {
 
 #elif defined(TD_MSG_DICT_)
 
-#undef TD_NEW_MSG_SEG
+  #undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
+  #undef TD_DEF_MSG_TYPE
-#define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
+  #undef TD_CLOSE_MSG_TYPE
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
+  #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
+  #define TD_CLOSE_MSG_TYPE(type)
+
+  int32_t tMsgDict[] = {
 
-int32_t tMsgDict[] = {
 
 #elif defined(TD_MSG_SEG_CODE_)
 
-#undef TD_NEW_MSG_SEG
+  #undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
+  #undef TD_DEF_MSG_TYPE
-#define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE,
+  #undef TD_CLOSE_MSG_TYPE
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
+  #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE,
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
+  #define TD_CLOSE_MSG_TYPE(TYPE)
+
+  enum {
 
-enum {
 
 #else 
 
-#undef TD_NEW_MSG_SEG
+  #undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
+  #undef TD_DEF_MSG_TYPE
-#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
+  #undef TD_CLOSE_MSG_TYPE
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
+  #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
+  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
 
-enum { // WARN: new msg should be appended to segment tail
+  enum { // WARN: new msg should be appended to segment tail
 #endif
   TD_NEW_MSG_SEG(TDMT_DND_MSG)    // 0<<8
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL)
@@ -82,10 +104,12 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_DND_UNUSED_CODE, "dnd-unused", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
@@ -194,6 +218,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@@ -231,7 +256,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_EXEC_RSMA, "vnode-exec-rsma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DELETE, "delete-data", SVDeleteReq, SVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_BATCH_DEL, "batch-delete", SBatchDeleteReq, NULL)
-  TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
+TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_REPLICA, "alter-replica", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIRM, "alter-confirm", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
@@ -243,6 +268,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
@@ -257,6 +283,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
@@ -274,9 +301,11 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
@@ -308,6 +337,8 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) 
+
 
   TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL)
@@ -317,6 +348,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
@@ -330,9 +362,15 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
+  TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) 
+
+  TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
   
 
-#if defined(TD_MSG_NUMBER_)
+  
+
+
+  #if defined(TD_MSG_NUMBER_)
     TDMT_MAX
-#endif
+  #endif
 };

include/common/ttypes.h

@@ -275,9 +275,11 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT64(_t)     ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
+#define IS_VALID_UINT64(_t)    ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -37,6 +37,10 @@ typedef struct SVariant {
   };
 } SVariant;
 
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value);
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value);
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 

include/libs/executor/storageapi.h

@@ -152,6 +152,18 @@ typedef struct {
 
 // clang-format off
 /*-------------------------------------------------new api format---------------------------------------------------*/
+typedef enum {
+  TSD_READER_NOTIFY_DURATION_START
+} ETsdReaderNotifyType;
+
+typedef union {
+  struct {
+    int32_t filesetId;
+  } duration;
+} STsdReaderNotifyInfo;
+
+typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param);
+
 typedef struct TsdReader {
   int32_t      (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
                            SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
@@ -169,6 +181,9 @@ typedef struct TsdReader {
   int32_t      (*tsdReaderGetDataBlockDistInfo)();
   int64_t      (*tsdReaderGetNumOfInMemRows)();
   void         (*tsdReaderNotifyClosing)();
+
+  void         (*tsdSetFilesetDelimited)(void* pReader);
+  void         (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 } TsdReader;
 
 typedef struct SStoreCacheReader {
@@ -350,6 +365,8 @@ typedef struct SStateStore {
   TSKEY        (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
   bool         (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
   bool         (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
+  bool         (*isIncrementalTimeStamp)(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
+
   void (*updateInfoDestroy)(SUpdateInfo* pInfo);
   void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count);
   void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count);

include/libs/nodes/plannodes.h

@@ -118,6 +118,7 @@ typedef struct SScanLogicNode {
   bool          igLastNull;
   bool          groupOrderScan;
   bool          onlyMetaCtbIdx; // for tag scan with no tbname
+  bool          filesetDelimited; // returned blocks delimited by fileset
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -432,6 +433,7 @@ typedef struct STableScanPhysiNode {
   int8_t         igExpired;
   bool           assignBlockUid;
   int8_t         igCheckUpdate;
+  bool           filesetDelimited;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;

include/libs/stream/tstream.h

@@ -216,9 +216,6 @@ int32_t         streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem);
 SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue);
 #endif
 
-int32_t streamInit();
-void    streamCleanUp();
-
 SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type);
 void               streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
 

include/libs/stream/tstreamUpdate.h

@@ -55,6 +55,7 @@ int32_t      updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *p
 int32_t      updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
 void         windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
 void         windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
+bool         isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -558,7 +558,6 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_GRANT_GEN_IVLD_KEY            TAOS_DEF_ERROR_CODE(0, 0x0812)
 #define TSDB_CODE_GRANT_GEN_APP_LIMIT           TAOS_DEF_ERROR_CODE(0, 0x0813)
 #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN       TAOS_DEF_ERROR_CODE(0, 0x0814)
-#define TSDB_CODE_GRANT_PAR_IVLD_DIST           TAOS_DEF_ERROR_CODE(0, 0x0815)
 
 // sync
 // #define TSDB_CODE_SYN_INVALID_CONFIG         TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

include/util/tdef.h

@@ -305,7 +305,7 @@ typedef enum ELogicConditionType {
 #define TSDB_SYNC_APPLYQ_SIZE_LIMIT    512
 #define TSDB_SYNC_NEGOTIATION_WIN      512
 
-#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048
+#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024
 
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta

source/client/inc/clientSml.h

@@ -65,14 +65,32 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-//#define TS        "_ts"
-//#define TS_LEN    3
 #define VALUE     "_value"
-#define VALUE_LEN 6
+#define VALUE_LEN (sizeof(VALUE)-1)
 
 #define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 10
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
+
+#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+
+#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
+
+#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
+
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
 
 typedef enum {
   SCHEMA_ACTION_NULL,
@@ -83,18 +101,6 @@ typedef enum {
   SCHEMA_ACTION_CHANGE_TAG_SIZE,
 } ESchemaAction;
 
-typedef struct {
-  const void  *key;
-  int32_t      keyLen;
-  void        *value;
-  bool         used;
-}Node;
-
-typedef struct NodeList{
-  Node             data;
-  struct NodeList* next;
-}NodeList;
-
 typedef struct {
   char *measure;
   char *tags;
@@ -117,7 +123,6 @@ typedef struct {
   int32_t     sTableNameLen;
   char        childTableName[TSDB_TABLE_NAME_LEN];
   uint64_t    uid;
-//  void       *key;        // for openTsdb
 
   SArray *tags;
 
@@ -161,7 +166,8 @@ typedef struct {
 typedef struct {
   int64_t id;
 
-  SMLProtocolType protocol;
+  TSDB_SML_PROTOCOL_TYPE protocol;
+
   int8_t          precision;
   bool            reRun;
   bool            dataFormat;  // true means that the name and order of keys in each line are the same(only for influx protocol)
@@ -201,29 +207,8 @@ typedef struct {
   bool         needModifySchema;
 } SSmlHandle;
 
-#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+extern int64_t smlFactorNS[];
-&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+extern int64_t smlFactorS[];
-
-#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
-
-#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
-
-#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
-
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
-
-#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
-  for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
-      MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      keyLen--;                              \
-    }                                        \
-  }
-
-extern int64_t smlFactorNS[3];
-extern int64_t smlFactorS[3];
 
 typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
@@ -231,16 +216,10 @@ SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
 int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-//int           nodeListSize(NodeList* list);
-bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
 int64_t       smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
-int8_t        smlGetTsTypeByLen(int32_t len);
 SSmlTableInfo*    smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
 SSmlSTableMeta*   smlBuildSTableMeta(bool isDataFormat);
 int32_t           smlSetCTableName(SSmlTableInfo *oneTable);
@@ -258,7 +237,40 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
-void    smlStrReplace(char* src, int32_t len);
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
+bool            isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+bool            isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+int32_t         smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlJoinMeasureTag(SSmlLineInfo *elements);
+void            smlBuildTsKv(SSmlKv *kv, int64_t ts);
+int32_t         smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
+int32_t         smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
+
+static inline bool smlDoubleToInt64OverFlow(double num) {
+  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
+  return false;
+}
+
+static inline void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
+static inline int8_t smlGetTsTypeByLen(int32_t len) {
+  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
+    return TSDB_TIME_PRECISION_SECONDS;
+  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+    return TSDB_TIME_PRECISION_MILLI;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -732,7 +732,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         return -1;
       }
       newstr[0] = '"';
-      strncpy(newstr+1, str, len);
+      memcpy(newstr+1, str, len);
       newstr[len + 1] = '"';
       newstr[len + 2] = '\0';
       str = newstr;

source/client/src/clientSml.c

@@ -20,75 +20,93 @@
 
 #include "clientSml.h"
 
-int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
+#define RETURN_FALSE                                 \
-int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
+  return false;
 
-//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
+#define SET_DOUBLE                     \
-//  NodeList *tmp = list;
+  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-//  while (tmp) {
+  kvVal->d = result;
-//    if (fn == NULL) {
+
-//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+#define SET_FLOAT                                                                              \
-//        return tmp->data.value;
+  if (!IS_VALID_FLOAT(result)) {                                                               \
-//      }
+    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-//    } else {
+    return false;                                                                              \
-//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
+  }                                                                                            \
-//        return tmp->data.value;
+  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-//      }
+  kvVal->f = (float)result;
-//    }
+
-//
+#define SET_BIGINT                                                                                       \
-//    tmp = tmp->next;
+  errno = 0;                                                                                             \
-//  }
+  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-//  return NULL;
+  if (errno == ERANGE) {                                                                                 \
-//}
+    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-//
+    return false;                                                                                        \
-//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
+  }                                                                                                      \
-//  NodeList *tmp = *list;
+  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-//  while (tmp) {
+  kvVal->i = tmp;
-//    if (!tmp->data.used) break;
+
-//    if (fn == NULL) {
+#define SET_INT                                                                    \
-//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
+  if (!IS_VALID_INT(result)) {                                                     \
-//        return -1;
+    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-//      }
+    return false;                                                                  \
-//    } else {
+  }                                                                                \
-//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
+  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-//        return -1;
+  kvVal->i = result;
-//      }
+
-//    }
+#define SET_SMALL_INT                                                          \
-//
+  if (!IS_VALID_SMALLINT(result)) {                                            \
-//    tmp = tmp->next;
+    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-//  }
+    return false;                                                              \
-//  if (tmp) {
+  }                                                                            \
-//    tmp->data.key = key;
+  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-//    tmp->data.keyLen = len;
+  kvVal->i = result;
-//    tmp->data.value = value;
+
-//    tmp->data.used = true;
+#define SET_UBIGINT                                                                             \
-//  } else {
+  errno = 0;                                                                                    \
-//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
+  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-//    if (newNode == NULL) {
+  if (errno == ERANGE || result < 0) {                                                          \
-//      return -1;
+    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-//    }
+    return false;                                                                               \
-//    newNode->data.key = key;
+  }                                                                                             \
-//    newNode->data.keyLen = len;
+  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-//    newNode->data.value = value;
+  kvVal->u = tmp;
-//    newNode->data.used = true;
+
-//    newNode->next = *list;
+#define SET_UINT                                                                  \
-//    *list = newNode;
+  if (!IS_VALID_UINT(result)) {                                                   \
-//  }
+    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-//  return 0;
+    return false;                                                                 \
-//}
+  }                                                                               \
-//
+  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-//int nodeListSize(NodeList *list) {
+  kvVal->u = result;
-//  int cnt = 0;
+
-//  while (list) {
+#define SET_USMALL_INT                                                            \
-//    if (list->data.used)
+  if (!IS_VALID_USMALLINT(result)) {                                              \
-//      cnt++;
+    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-//    else
+    return false;                                                                 \
-//      break;
+  }                                                                               \
-//    list = list->next;
+  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-//  }
+  kvVal->u = result;
-//  return cnt;
+
-//}
+#define SET_TINYINT                                                       \
+  if (!IS_VALID_TINYINT(result)) {                                        \
+    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
+    return false;                                                         \
+  }                                                                       \
+  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
+  kvVal->i = result;
+
+#define SET_UTINYINT                                                            \
+  if (!IS_VALID_UTINYINT(result)) {                                             \
+    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
+    return false;                                                               \
+  }                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
+  kvVal->u = result;
+
+int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
+int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
 
 static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
   SUserAuthInfo pAuth = {0};
@@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
-inline bool smlDoubleToInt64OverFlow(double num) {
-  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
-  return false;
-}
-
-void smlStrReplace(char* src, int32_t len){
-  if (!tsSmlDot2Underline) return;
-  for(int i = 0; i < len; i++){
-    if(src[i] == '.'){
-      src[i] = '_';
-    }
-  }
-}
 
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
@@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
   return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
 }
 
-int8_t smlGetTsTypeByLen(int32_t len) {
-  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
-    return TSDB_TIME_PRECISION_SECONDS;
-  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-    return TSDB_TIME_PRECISION_MILLI;
-  } else {
-    return -1;
-  }
-}
-
 SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
   SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
   if (!tag) {
@@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu
     goto cleanup;
   }
 
-  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
-  //  if (tag->tags == NULL) {
-  //    uError("SML:smlBuildTableInfo failed to allocate memory");
-  //    goto cleanup;
-  //  }
   return tag;
 
 cleanup:
@@ -192,6 +182,242 @@ cleanup:
   return NULL;
 }
 
+void smlBuildTsKv(SSmlKv *kv, int64_t ts){
+  kv->key = tsSmlTsDefaultName;
+  kv->keyLen = strlen(tsSmlTsDefaultName);
+  kv->type = TSDB_DATA_TYPE_TIMESTAMP;
+  kv->i = ts;
+  kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){
+  SSmlSTableMeta* sMeta = NULL;
+  char *measure = currElement->measure;
+  int   measureLen = currElement->measureLen;
+  if (currElement->measureEscaped) {
+    measure = (char *)taosMemoryMalloc(measureLen);
+    memcpy(measure, currElement->measure, measureLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+  }
+  STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+  if (currElement->measureEscaped) {
+    taosMemoryFree(measure);
+  }
+  if (pTableMeta == NULL) {
+    info->dataFormat = false;
+    info->reRun = true;
+    terrno = TSDB_CODE_SUCCESS;
+    return sMeta;
+  }
+  sMeta = smlBuildSTableMeta(info->dataFormat);
+  if(sMeta == NULL){
+    taosMemoryFreeClear(pTableMeta);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return sMeta;
+  }
+  sMeta->tableMeta = pTableMeta;
+  taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+  for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
+    SSchema *col = pTableMeta->schema + i;
+    SSmlKv   kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
+    if (col->type == TSDB_DATA_TYPE_NCHAR) {
+      kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) {
+      kv.length = col->bytes - VARSTR_HEADER_SIZE;
+    } else{
+      kv.length = col->bytes;
+    }
+
+    if(i < pTableMeta->tableInfo.numOfColumns){
+      taosArrayPush(sMeta->cols, &kv);
+    }else{
+      taosArrayPush(sMeta->tags, &kv);
+    }
+  }
+  return sMeta;
+}
+
+bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  // cnt begin 0, add ts so + 2
+  if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  // bind data
+  int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
+  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+    uDebug("smlBuildCol error, retry");
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  if (cnt >= taosArrayGetSize(info->maxColKVs)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+
+  if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+}
+
+bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+    goto END;
+  }
+
+  if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    goto END;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    goto END;
+  }
+
+  if (unlikely(kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+
+END:
+  info->dataFormat = false;
+  info->reRun = true;
+  return false;
+}
+
+int32_t smlJoinMeasureTag(SSmlLineInfo *elements){
+  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+  if(elements->measureTag == NULL){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
+  bool isSameMeasure = IS_SAME_SUPER_TABLE;
+  if(isSameMeasure) {
+    return 0;
+  }
+  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+
+  SSmlSTableMeta *sMeta = NULL;
+  if (unlikely(tmp == NULL)) {
+    sMeta = smlBuildSuperTableInfo(info, elements);
+    if(sMeta == NULL) return -1;
+  }else{
+    sMeta = *tmp;
+  }
+  ASSERT(sMeta != NULL);
+  info->currSTableMeta = sMeta->tableMeta;
+  info->maxTagKVs = sMeta->tags;
+  info->maxColKVs = sMeta->cols;
+  return 0;
+}
+
+int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
+  SSmlTableInfo **oneTable =
+      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
+  SSmlTableInfo *tinfo = NULL;
+  if (unlikely(oneTable == NULL)) {
+    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
+    if (unlikely(!tinfo)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES);
+
+    tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
+    for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
+      SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
+      if (kv->keyEscaped) kv->key = NULL;
+      if (kv->valueEscaped) kv->value = NULL;
+    }
+
+    smlSetCTableName(tinfo);
+    getTableUid(info, elements, tinfo);
+    if (info->dataFormat) {
+      info->currSTableMeta->uid = tinfo->uid;
+      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
+      if (tinfo->tableDataCtx == NULL) {
+        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
+        smlDestroyTableInfo(&tinfo);
+        return TSDB_CODE_SML_INVALID_DATA;
+      }
+    }
+  }else{
+    tinfo = *oneTable;
+  }
+  ASSERT(tinfo != NULL);
+  if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1);
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i);
+    if (elements->colArray == NULL) {
+      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
+    }
+    taosArrayPush(elements->colArray, kvTs);
+    taosArrayPush(elements->colArray, kv);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i);
+    taosArraySet(elements->colArray, 0, kvTs);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
   bool autoChildName = false;
   size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
@@ -328,98 +554,6 @@ cleanup:
   return NULL;
 }
 
-// uint16_t smlCalTypeSum(char* endptr, int32_t left){
-//   uint16_t sum = 0;
-//   for(int i = 0; i < left; i++){
-//     sum += endptr[i];
-//   }
-//   return sum;
-// }
-
-#define RETURN_FALSE                                 \
-  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-  return false;
-
-#define SET_DOUBLE                     \
-  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-  kvVal->d = result;
-
-#define SET_FLOAT                                                                              \
-  if (!IS_VALID_FLOAT(result)) {                                                               \
-    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-    return false;                                                                              \
-  }                                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-  kvVal->f = (float)result;
-
-#define SET_BIGINT                                                                                       \
-  errno = 0;                                                                                             \
-  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-  if (errno == ERANGE) {                                                                                 \
-    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-    return false;                                                                                        \
-  }                                                                                                      \
-  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-  kvVal->i = tmp;
-
-#define SET_INT                                                                    \
-  if (!IS_VALID_INT(result)) {                                                     \
-    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-    return false;                                                                  \
-  }                                                                                \
-  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-  kvVal->i = result;
-
-#define SET_SMALL_INT                                                          \
-  if (!IS_VALID_SMALLINT(result)) {                                            \
-    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-    return false;                                                              \
-  }                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-  kvVal->i = result;
-
-#define SET_UBIGINT                                                                             \
-  errno = 0;                                                                                    \
-  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-  if (errno == ERANGE || result < 0) {                                                          \
-    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-    return false;                                                                               \
-  }                                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-  kvVal->u = tmp;
-
-#define SET_UINT                                                                  \
-  if (!IS_VALID_UINT(result)) {                                                   \
-    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-  kvVal->u = result;
-
-#define SET_USMALL_INT                                                            \
-  if (!IS_VALID_USMALLINT(result)) {                                              \
-    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-  kvVal->u = result;
-
-#define SET_TINYINT                                                       \
-  if (!IS_VALID_TINYINT(result)) {                                        \
-    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
-    return false;                                                         \
-  }                                                                       \
-  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
-  kvVal->i = result;
-
-#define SET_UTINYINT                                                            \
-  if (!IS_VALID_UTINYINT(result)) {                                             \
-    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
-    return false;                                                               \
-  }                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
-  kvVal->u = result;
-
 bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
   const char *pVal = kvVal->value;
   int32_t     len = kvVal->length;
@@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
 static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                               ESchemaAction action) {
   SRequestObj   *pRequest = NULL;
@@ -1121,35 +1253,6 @@ end:
   return code;
 }
 
-/*
-static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
-  for(int i = 0; i < taosArrayGetSize(tags); i++) {
-    SSmlKv *tag = taosArrayGet(tags, i);
-    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
-      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
-      return TSDB_CODE_TSC_DUP_NAMES;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
-  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-
-  end:
-  taosHashCleanup(dumplicateKey);
-  return ret;
-}
-*/
-
 static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) {
     taosHashCleanup(kvHash);
   }
 
-  //  if (info->parseJsonByLib) {
-  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
-  //    if (key != NULL) taosMemoryFree(key->tags);
-  //  }
-  //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
@@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
 
-  // destroy info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // destroy info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
-
-  // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
   taosHashCleanup(info->childTables);
   taosHashCleanup(info->superTables);
@@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) {
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
-      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+      if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) {
+        taosMemoryFree(info->lines[i].measureTag);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
         return ret;
       }
     } else {
-      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
-      //      if (ret != TSDB_CODE_SUCCESS) {
-      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
-      //        return ret;
-      //      }
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
@@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
   uDebug(
       "SML:0x%" PRIx64
       " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
-        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
+        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64,
-      "",
       info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
       info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
       info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
       info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
       info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
+
 }
 
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
-  // clear info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(info, *oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // clear info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);

source/client/src/clientSmlJson.c

@@ -29,199 +29,6 @@
       (start)++;                 \
   }
 
-// SArray *smlJsonParseTags(char *start, char *end){
-//   SArray *tags = taosArrayInit(4, sizeof(SSmlKv));
-//   while(start < end){
-//     SSmlKv kv = {0};
-//     kv.type = TSDB_DATA_TYPE_NCHAR;
-//     bool isInQuote = false;
-//     while(start < end){
-//       if(unlikely(!isInQuote && *start == '"')){
-//         start++;
-//         kv.key = start;
-//         isInQuote = true;
-//         continue;
-//       }
-//       if(unlikely(isInQuote && *start == '"')){
-//         kv.keyLen = start - kv.key;
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//     bool hasColon = false;
-//     while(start < end){
-//       if(unlikely(!hasColon && *start == ':')){
-//         start++;
-//         hasColon = true;
-//         continue;
-//       }
-//       if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){
-//         kv.value = start;
-//         start++;
-//         continue;
-//       }
-//
-//       if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){
-//         kv.length = start - kv.value;
-//         taosArrayPush(tags, &kv);
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//   }
-//   return tags;
-// }
-
-// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) {
-//   int32_t ret = TSDB_CODE_SUCCESS;
-//
-//   if(is_same_child_table_telnet(elements, &info->preLine) == 0){
-//     return TSDB_CODE_SUCCESS;
-//   }
-//
-//   bool isSameMeasure = IS_SAME_SUPER_TABLE;
-//
-//   int     cnt = 0;
-//   SArray *preLineKV = info->preLineTagKV;
-//   bool    isSuperKVInit = true;
-//   SArray *superKV = NULL;
-//   if(info->dataFormat){
-//     if(unlikely(!isSameMeasure)){
-//       SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//       elements->measureLen, NULL);
-//
-//       if(unlikely(sMeta == NULL)){
-//         sMeta = smlBuildSTableMeta(info->dataFormat);
-//         STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-//         sMeta->tableMeta = pTableMeta;
-//         if(pTableMeta == NULL){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
-//       }
-//       info->currSTableMeta = sMeta->tableMeta;
-//       superKV = sMeta->tags;
-//
-//       if(unlikely(taosArrayGetSize(superKV) == 0)){
-//         isSuperKVInit = false;
-//       }
-//       taosArraySetSize(preLineKV, 0);
-//     }
-//   }else{
-//     taosArraySetSize(preLineKV, 0);
-//   }
-//
-//   SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen);
-//   int32_t tagNum = taosArrayGetSize(tags);
-//   if (tagNum == 0) {
-//     uError("SML:tag is empty:%s", elements->tags)
-//     taosArrayDestroy(tags);
-//     return TSDB_CODE_SML_INVALID_DATA;
-//   }
-//   for (int32_t i = 0; i < tagNum; ++i) {
-//     SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i);
-//
-//     if(info->dataFormat){
-//       if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){
-//         info->dataFormat = false;
-//         info->reRun      = true;
-//         taosArrayDestroy(tags);
-//         return TSDB_CODE_SUCCESS;
-//       }
-//
-//       if(isSameMeasure){
-//         if(unlikely(cnt >= taosArrayGetSize(preLineKV))) {
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-//         if(unlikely(kv.length > preKV->length)){
-//           preKV->length = kv.length;
-//           SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//           elements->measureLen, NULL);
-//            if(unlikely(NULL == tableMeta)){
-//              uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-//              return TSDB_CODE_SML_INTERNAL_ERROR;
-//            }
-//
-//           SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
-//           oldKV->length = kv.length;
-//           info->needModifySchema = true;
-//         }
-//         if(unlikely(!IS_SAME_KEY)){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//       }else{
-//         if(isSuperKVInit){
-//           if(unlikely(cnt >= taosArrayGetSize(superKV))) {
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//           SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-//           if(unlikely(kv.length > preKV->length)) {
-//             preKV->length = kv.length;
-//           }else{
-//             kv.length = preKV->length;
-//           }
-//           info->needModifySchema = true;
-//
-//           if(unlikely(!IS_SAME_KEY)){
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//         }else{
-//           taosArrayPush(superKV, &kv);
-//         }
-//         taosArrayPush(preLineKV, &kv);
-//       }
-//     }else{
-//       taosArrayPush(preLineKV, &kv);
-//     }
-//     cnt++;
-//   }
-//   taosArrayDestroy(tags);
-//
-//   SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES,
-//   is_same_child_table_telnet); if (unlikely(tinfo == NULL)) {
-//     tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-//     if (unlikely(!tinfo)) {
-//       return TSDB_CODE_OUT_OF_MEMORY;
-//     }
-//     tinfo->tags = taosArrayDup(preLineKV, NULL);
-//
-//     smlSetCTableName(tinfo);
-//     if (info->dataFormat) {
-//       info->currSTableMeta->uid = tinfo->uid;
-//       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-//       if (tinfo->tableDataCtx == NULL) {
-//         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-//         return TSDB_CODE_SML_INVALID_DATA;
-//       }
-//     }
-//
-//     SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-//     *key = *elements;
-//     tinfo->key = key;
-//     nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
-//   }
-//   if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
-//
-//   return ret;
-// }
-
 static char *smlJsonGetObj(char *payload) {
   int  leftBracketCnt = 0;
   bool isInQuote = false;
@@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
       break;
     }
     case cJSON_String: {
-      /* set default JSON type to binary/nchar according to
+      smlConvertJSONString(kv, "binary", root);
-       * user configured parameter tsDefaultJSONStrType
-       */
-
-      char *tsDefaultJSONStrType = "binary";  // todo
-      smlConvertJSONString(kv, tsDefaultJSONStrType, root);
       break;
     }
     case cJSON_Object: {
@@ -682,138 +484,70 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
+static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
-  int32_t ret = TSDB_CODE_SUCCESS;
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
+  taosArrayClearEx(preLineKV, freeSSmlKv);
-    if (unlikely(!isSameMeasure)) {
+  int     cnt = 0;
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
-  taosArrayClear(preLineKV);
 
   int32_t tagNum = cJSON_GetArraySize(tags);
   if (unlikely(tagNum == 0)) {
     uError("SML:Tag should not be empty");
-    return TSDB_CODE_TSC_INVALID_JSON;
+    terrno = TSDB_CODE_TSC_INVALID_JSON;
+    return -1;
   }
   for (int32_t i = 0; i < tagNum; ++i) {
     cJSON *tag = cJSON_GetArrayItem(tags, i);
     if (unlikely(tag == NULL)) {
-      return TSDB_CODE_TSC_INVALID_JSON;
+      terrno = TSDB_CODE_TSC_INVALID_JSON;
+      return -1;
     }
-    //    if(unlikely(tag == cMeasure)) continue;
     size_t keyLen = strlen(tag->string);
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
       uError("OTD:Tag key length is 0 or too large than 64");
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno =  TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // add kv to SSmlKv
-    SSmlKv kv = {.key = tag->string, .keyLen = keyLen};
+    SSmlKv kv = {0};
+    kv.key = tag->string;
+    kv.keyLen = keyLen;
+
     // value
-    ret = smlParseValueFromJSON(tag, &kv);
+    int32_t ret = smlParseValueFromJSON(tag, &kv);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      return ret;
+      terrno =  ret;
-    }
+      return -1;
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
     }
     taosArrayPush(preLineKV, &kv);
+
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno =  TSDB_CODE_SUCCESS;
+      return -1;
+    }
+
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  int32_t ret = 0;
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  if(info->dataFormat){
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+    ret = smlProcessSuperTable(info, elements);
-
+    if(ret != 0){
-  SSmlTableInfo **tmp =
+      return terrno;
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (unlikely(!tinfo)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
     }
   }
-
+  ret = smlProcessTagJson(info, tags);
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
+  if(ret != 0){
-    //    *key = *elements;
+    return terrno;
-    //    if(info->parseJsonByLib){
-    //      key->tags = taosMemoryMalloc(elements->tagsLen + 1);
-    //      memcpy(key->tags, elements->tags, elements->tagsLen);
-    //      key->tags[elements->tagsLen] = 0;
-    //    }
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
+  ret = smlJoinMeasureTag(elements);
-
+  if(ret != 0){
     return ret;
+  }
+  return smlProcessChildTable(info, elements);
 }
 
 static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
@@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
+  SSmlKv kvTs = {0};
-                 .keyLen = strlen(tsSmlTsDefaultName),
+  smlBuildTsKv(&kvTs, ts);
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
-  if (info->dataFormat) {
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
 }
 
 static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
@@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-
   if (unlikely(info->lines != NULL)) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
@@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
+  SSmlKv kvTs = {0};
-                 .keyLen = strlen(tsSmlTsDefaultName),
+  smlBuildTsKv(&kvTs, ts);
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
-  if (info->dataFormat) {
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
 }
 
 int32_t smlParseJSON(SSmlHandle *info, char *payload) {

source/client/src/clientSmlLine.c

@@ -20,15 +20,9 @@
 
 #include "clientSml.h"
 
-// comma ,
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-// space
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-// equal =
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
-// quote "
-// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-// SLASH
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
@@ -51,10 +45,10 @@
     }                                                \
   }
 
-#define BINARY_ADD_LEN 2  // "binary"   2 means " "
+#define BINARY_ADD_LEN (sizeof("\"\"")-1)    // "binary"   2 means length of ("")
-#define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
+#define NCHAR_ADD_LEN  (sizeof("L\"\"")-1)   // L"nchar"   3 means length of (L"")
 
-uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
+uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
                                   TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO,
                                   TSDB_TIME_PRECISION_NANO};
 
@@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   return TSDB_CODE_TSC_INVALID_VALUE;
 }
 
-static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
+static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
-                             bool isSameCTable) {
-  if (isSameCTable) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
-        for (int i = pTableMeta->tableInfo.numOfColumns;
-             i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name,
-                         .keyLen = strlen(tag->name),
-                         .type = tag->type,
-                         .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE};
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
   taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
@@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     while (*sql < sqlEnd) {
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(IS_EQUAL(*sql))) {
         keyLen = *sql - key;
@@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // parse value
@@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         break;
       } else if (unlikely(IS_EQUAL(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(valueLen == 0)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_SML_INVALID_DATA;
+      terrno = TSDB_CODE_SML_INVALID_DATA;
+      return -1;
     }
 
     if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
     if (keyEscaped) {
@@ -338,30 +286,10 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         .keyEscaped = keyEscaped,
         .valueEscaped = valueEscaped};
     taosArrayPush(preLineKV, &kv);
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
 
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
-        info->dataFormat = false;
+      terrno = TSDB_CODE_SUCCESS;
-        info->reRun = true;
+      return -1;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
     }
 
     cnt++;
@@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
     (*sql)++;
   }
-
+  return 0;
-  void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-  if ((oneTable != NULL)) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen);
-  if (unlikely(!tinfo)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tinfo->tags = taosArrayDup(preLineKV, NULL);
-  for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
-    if (kv->keyEscaped) kv->key = NULL;
-    if (kv->valueEscaped) kv->value = NULL;
-  }
-
-  smlSetCTableName(tinfo);
-  getTableUid(info, currElement, tinfo);
-  if (info->dataFormat) {
-    info->currSTableMeta->uid = tinfo->uid;
-    tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-    if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(&tinfo);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-      return TSDB_CODE_SML_INVALID_DATA;
-    }
-  }
-
-  taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
-
-  return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
+static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
-                             bool isSameCTable) {
+  bool isSameCTable = IS_SAME_CHILD_TABLE;
-  int cnt = 0;
+  if(isSameCTable){
-  if (info->dataFormat) {
-    if (unlikely(!isSameCTable)) {
-      SSmlTableInfo **oneTable =
-          (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-      if (unlikely(oneTable == NULL)) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-      info->currTableDataCtx = (*oneTable)->tableDataCtx;
-    }
-
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
     return TSDB_CODE_SUCCESS;
   }
-        *tmp = smlBuildSTableMeta(info->dataFormat);
-        if(*tmp == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
 
-        for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
+  int32_t ret = 0;
-          SSchema *tag = pTableMeta->schema + i;
+  if(info->dataFormat){
-          SSmlKv   kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
+    ret = smlProcessSuperTable(info, elements);
-          if (tag->type == TSDB_DATA_TYPE_NCHAR) {
+    if(ret != 0){
-            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+      return terrno;
-          } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
-            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
-          }
-          taosArrayPush((*tmp)->cols, &kv);
-        }
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxColKVs = (*tmp)->cols;
     }
   }
 
+  ret = smlProcessTagLine(info, sql, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  return smlProcessChildTable(info, elements);
+}
+
+static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
+  int cnt = 0;
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
@@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
 
     if (info->dataFormat) {
-      // cnt begin 0, add ts so + 2
+      bool isAligned = isSmlColAligned(info, cnt, &kv);
-      if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
-        info->dataFormat = false;
-        info->reRun = true;
       freeSSmlKv(&kv);
+      if(!isAligned){
         return TSDB_CODE_SUCCESS;
       }
-      // bind data
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1);
-      if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-        uDebug("smlBuildCol error, retry");
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (cnt >= taosArrayGetSize(info->maxColKVs)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
-      if (kv.type != maxKV->type) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-      freeSSmlKv(&kv);
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA;
   elements->measure = sql;
-
   // parse measure
   size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
@@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
-
+  elements->measureTag = elements->measure;
-  bool isSameCTable = false;
-  bool isSameMeasure = false;
-  if (IS_SAME_CHILD_TABLE) {
-    isSameCTable = true;
-    isSameMeasure = true;
-  } else if (info->dataFormat) {
-    isSameMeasure = IS_SAME_SUPER_TABLE;
-  }
   // parse tag
   if (*sql == COMMA) sql++;
   elements->tags = sql;
 
-  int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  int ret = smlParseTagLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   elements->cols = sql;
 
-  ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  ret = smlParseColLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  // add ts to
-  SSmlKv kv = {.key = tsSmlTsDefaultName,
-               .keyLen = strlen(tsSmlTsDefaultName),
-               .type = TSDB_DATA_TYPE_TIMESTAMP,
-               .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
-               .keyEscaped = false,
-               .valueEscaped = false};
-  if (info->dataFormat) {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
 
-    clearColValArraySml(info->currTableDataCtx->pValues);
+  SSmlKv kvTs = {0};
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+  smlBuildTsKv(&kvTs, ts);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
-    taosArraySet(elements->colArray, 0, &kv);
-  }
-  info->preLine = *elements;
 
-  return ret;
+  return smlParseEndLine(info, elements, &kvTs);
 }

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,6 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b) {
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  //  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-  //         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
     return 1;
   return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
@@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
   }
 }
 
-static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
+static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
-  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-    elements->measureTag = info->preLine.measureTag;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
+  taosArrayClearEx(preLineKV, freeSSmlKv);
-    if (!isSameMeasure) {
+  int     cnt = 0;
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
 
-  taosArrayClear(preLineKV);
   const char *sql = data;
   while (sql < sqlEnd) {
     JUMP_SPACE(sql, sqlEnd)
@@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     // parse key
     while (sql < sqlEnd) {
       if (unlikely(*sql == SPACE)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(*sql == EQUAL)) {
         keyLen = sql - key;
@@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     }
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
-      smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
-    //    if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) {
-    //      smlBuildInvalidDataMsg(msg, "dumplicate key", key);
-    //      return TSDB_CODE_TSC_DUP_NAMES;
-    //    }
 
     // parse value
     const char *value = sql;
@@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         break;
       }
       if (unlikely(*sql == EQUAL)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       sql++;
     }
     valueLen = sql - value;
 
     if (unlikely(valueLen == 0)) {
-      smlBuildInvalidDataMsg(msg, "invalid value", value);
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      terrno = TSDB_CODE_TSC_INVALID_VALUE;
+      return -1;
     }
 
     if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
+    SSmlKv kv = {.key = key,
-
+        .keyLen = keyLen,
-    if (info->dataFormat) {
+        .type = TSDB_DATA_TYPE_NCHAR,
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+        .value = value,
-        info->dataFormat = false;
+        .length = valueLen,
-        info->reRun = true;
+        .keyEscaped = false,
-        return TSDB_CODE_SUCCESS;
+        .valueEscaped = false};
-      }
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-    }
     taosArrayPush(preLineKV, &kv);
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
+    }
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+    elements->measureTag = info->preLine.measureTag;
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (!tinfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
-  }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
     return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
+    }
+  }
+
+  ret = smlProcessTagTelnet(info, data, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+
+  return smlProcessChildTable(info, elements);
 }
 
 // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
@@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
-  bool needConverTime = false;  // get TS before parse tag(get meta), so need conver time
+  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
   if (info->dataFormat && info->currSTableMeta == NULL) {
     needConverTime = true;
   }
@@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
   // parse value
   smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
@@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf);
+  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SUCCESS;
   }
 
-  if (info->dataFormat && info->currSTableMeta != NULL) {
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
   if (needConverTime) {
     kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
 
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }

source/common/src/tdataformat.c

@@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
     n += tPutCStr(p ? p + n : p, pTagVal->pKey);
   } else {
     n += tPutI16v(p ? p + n : p, pTagVal->cid);
+    ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
   }
 
   // type

source/common/src/tglobal.c

@@ -1282,7 +1282,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 static int32_t taosCheckGlobalCfg() {
   uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn);
   if (ipv4 == 0xffffffff) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    terrno = TSDB_CODE_RPC_FQDN_ERROR;
     uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr());
     return -1;
   }

source/common/src/tmsg.c

@@ -18,23 +18,32 @@
 
 #undef TD_MSG_NUMBER_
 #undef TD_MSG_DICT_
+#undef TD_MSG_RANGE_CODE_
 #define TD_MSG_INFO_
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
 #undef TD_MSG_NUMBER_
 #undef TD_MSG_INFO_
+#undef TD_MSG_RANGE_CODE_
 #define TD_MSG_DICT_
 #undef TD_MSG_SEG_CODE_
 #include "tmsgdef.h"
 
+#undef TD_MSG_NUMBER_
+#undef TD_MSG_INFO_
+#undef TD_MSG_DICT_
+#undef TD_MSG_SEG_CODE_
+#define TD_MSG_RANGE_CODE_
+#include "tmsgdef.h"
+
 #include "tlog.h"
 
 #define DECODESQL()                                                                 \
   do {                                                                              \
-    if(!tDecodeIsEnd(&decoder)){                                 \
+    if (!tDecodeIsEnd(&decoder)) {                                                  \
-      if(tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1;    \
+      if (tDecodeI32(&decoder, &pReq->sqlLen) < 0) return -1;                       \
-      if(pReq->sqlLen > 0){                                     \
+      if (pReq->sqlLen > 0) {                                                       \
         if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->sql, NULL) < 0) return -1; \
       }                                                                             \
     }                                                                               \
@@ -42,7 +51,7 @@
 
 #define ENCODESQL()                                                        \
   do {                                                                     \
-    if (pReq->sqlLen > 0 && pReq->sql != NULL){                  \
+    if (pReq->sqlLen > 0 && pReq->sql != NULL) {                           \
       if (tEncodeI32(&encoder, pReq->sqlLen) < 0) return -1;               \
       if (tEncodeBinary(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; \
     }                                                                      \
@@ -50,7 +59,7 @@
 
 #define FREESQL()                \
   do {                           \
-    if(pReq->sql != NULL){            \
+    if (pReq->sql != NULL) {     \
       taosMemoryFree(pReq->sql); \
     }                            \
     pReq->sql = NULL;            \
@@ -742,9 +751,7 @@ int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq)
   return 0;
 }
 
-void tFreeSMDropStbReq(SMDropStbReq *pReq) {
+void tFreeSMDropStbReq(SMDropStbReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAlterStbReq *pReq) {
   SEncoder encoder = {0};
@@ -1489,9 +1496,7 @@ int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq)
   return 0;
 }
 
-void tFreeSDropUserReq(SDropUserReq *pReq) {
+void tFreeSDropUserReq(SDropUserReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) {
   if (pIpWhiteList == NULL) return NULL;
@@ -1822,7 +1827,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   char *tb = taosHashIterate(pRsp->readTbs, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1837,7 +1842,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   tb = taosHashIterate(pRsp->writeTbs, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1852,7 +1857,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   tb = taosHashIterate(pRsp->alterTbs, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1867,7 +1872,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   tb = taosHashIterate(pRsp->readViews, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1882,7 +1887,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   tb = taosHashIterate(pRsp->writeViews, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1897,7 +1902,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   tb = taosHashIterate(pRsp->alterViews, NULL);
   while (tb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(tb, &keyLen);
+    void * key = taosHashGetKey(tb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1912,7 +1917,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
   int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
   while (useDb != NULL) {
     size_t keyLen = 0;
-    void  *key = taosHashGetKey(useDb, &keyLen);
+    void * key = taosHashGetKey(useDb, &keyLen);
     if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
     if (tEncodeCStr(pEncoder, key) < 0) return -1;
 
@@ -1954,8 +1959,8 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
   pRsp->alterViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
   if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL ||
-      pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL ||
+      pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL || pRsp->writeViews == NULL ||
-      pRsp->writeViews == NULL || pRsp->alterViews == NULL ||pRsp->useDbs == NULL) {
+      pRsp->alterViews == NULL || pRsp->useDbs == NULL) {
     goto _err;
   }
 
@@ -2219,7 +2224,7 @@ int32_t tDeserializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhit
   return 0;
 }
 
-int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) {
+int32_t tSerializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
@@ -2237,7 +2242,7 @@ int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteL
   return tlen;
 }
 
-int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) {
+int32_t tDeserializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhiteListRsp *pRsp) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -2257,9 +2262,7 @@ int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhit
   return 0;
 }
 
-void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp) {
+void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp *pRsp) { taosMemoryFree(pRsp->pWhiteLists); }
-  taosMemoryFree(pRsp->pWhiteLists);
-}
 
 int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -2288,13 +2291,9 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQno
   return 0;
 }
 
-void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq){
+void tFreeSMCreateQnodeReq(SMCreateQnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
-void    tFreeSDDropQnodeReq(SDDropQnodeReq* pReq) {
+void tFreeSDDropQnodeReq(SDDropQnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -2336,9 +2335,7 @@ int32_t tDeserializeSDropDnodeReq(void *buf, int32_t bufLen, SDropDnodeReq *pReq
   return 0;
 }
 
-void tFreeSDropDnodeReq(SDropDnodeReq *pReq) {
+void tFreeSDropDnodeReq(SDropDnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -2369,9 +2366,7 @@ int32_t tDeserializeSRestoreDnodeReq(void *buf, int32_t bufLen, SRestoreDnodeReq
   return 0;
 }
 
-void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) {
+void tFreeSRestoreDnodeReq(SRestoreDnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -2404,9 +2399,7 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq
   return 0;
 }
 
-void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) {
+void tFreeSMCfgDnodeReq(SMCfgDnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) {
   SEncoder encoder = {0};
@@ -2464,9 +2457,7 @@ int32_t tDeserializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *
   return 0;
 }
 
-void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) {
+void tFreeSCreateDnodeReq(SCreateDnodeReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq) {
   SEncoder encoder = {0};
@@ -3121,9 +3112,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) {
   return 0;
 }
 
-void tFreeSAlterDbReq(SAlterDbReq *pReq) {
+void tFreeSAlterDbReq(SAlterDbReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) {
   SEncoder encoder = {0};
@@ -3154,9 +3143,7 @@ int32_t tDeserializeSDropDbReq(void *buf, int32_t bufLen, SDropDbReq *pReq) {
   return 0;
 }
 
-void tFreeSDropDbReq(SDropDbReq *pReq) {
+void tFreeSDropDbReq(SDropDbReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSDropDbRsp(void *buf, int32_t bufLen, SDropDbRsp *pRsp) {
   SEncoder encoder = {0};
@@ -3435,9 +3422,7 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
   return 0;
 }
 
-void tFreeSCompactDbReq(SCompactDbReq *pReq) {
+void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
@@ -4611,9 +4596,7 @@ int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pR
   return 0;
 }
 
-void tFreeSMDropTopicReq(SMDropTopicReq *pReq) {
+void tFreeSMDropTopicReq(SMDropTopicReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSMDropCgroupReq(void *buf, int32_t bufLen, SMDropCgroupReq *pReq) {
   SEncoder encoder = {0};
@@ -5501,9 +5484,7 @@ int32_t tDeserializeSBalanceVgroupReq(void *buf, int32_t bufLen, SBalanceVgroupR
   return 0;
 }
 
-void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) {
+void tFreeSBalanceVgroupReq(SBalanceVgroupReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgroupLeaderReq *pReq) {
   SEncoder encoder = {0};
@@ -5526,7 +5507,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV
 
   if (tStartDecode(&decoder) < 0) return -1;
   if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
-  if(!tDecodeIsEnd(&decoder)){
+  if (!tDecodeIsEnd(&decoder)) {
     if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   }
 
@@ -5537,9 +5518,7 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV
   return 0;
 }
 
-void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) {
+void tFreeSBalanceVgroupLeaderReq(SBalanceVgroupLeaderReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSMergeVgroupReq(void *buf, int32_t bufLen, SMergeVgroupReq *pReq) {
   SEncoder encoder = {0};
@@ -5601,9 +5580,7 @@ int32_t tDeserializeSRedistributeVgroupReq(void *buf, int32_t bufLen, SRedistrib
   return 0;
 }
 
-void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) {
+void tFreeSRedistributeVgroupReq(SRedistributeVgroupReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSSplitVgroupReq(void *buf, int32_t bufLen, SSplitVgroupReq *pReq) {
   SEncoder encoder = {0};
@@ -7152,9 +7129,7 @@ int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq *
   return 0;
 }
 
-void tFreeMDropStreamReq(SMDropStreamReq *pReq) {
+void tFreeMDropStreamReq(SMDropStreamReq *pReq) { FREESQL(); }
-  FREESQL();
-}
 
 int32_t tSerializeSMRecoverStreamReq(void *buf, int32_t bufLen, const SMRecoverStreamReq *pReq) {
   SEncoder encoder = {0};
@@ -7297,8 +7272,8 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) {
   } else {
     ASSERT(0);
   }
-  //ENCODESQL
+  // ENCODESQL
-  if(pReq->sqlLen > 0 && pReq->sql != NULL) {
+  if (pReq->sqlLen > 0 && pReq->sql != NULL) {
     if (tEncodeI32(pCoder, pReq->sqlLen) < 0) return -1;
     if (tEncodeBinary(pCoder, pReq->sql, pReq->sqlLen) < 0) return -1;
   }
@@ -7345,11 +7320,11 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) {
     ASSERT(0);
   }
 
-  //DECODESQL
+  // DECODESQL
-  if(!tDecodeIsEnd(pCoder)){
+  if (!tDecodeIsEnd(pCoder)) {
-    if(tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1;
+    if (tDecodeI32(pCoder, &pReq->sqlLen) < 0) return -1;
-    if(pReq->sqlLen > 0){
+    if (pReq->sqlLen > 0) {
-      if (tDecodeBinaryAlloc(pCoder, (void**)&pReq->sql, NULL) < 0) return -1;
+      if (tDecodeBinaryAlloc(pCoder, (void **)&pReq->sql, NULL) < 0) return -1;
     }
   }
 
@@ -7375,7 +7350,7 @@ void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) {
     }
   }
 
-  if(pReq->sql != NULL){
+  if (pReq->sql != NULL) {
     taosMemoryFree(pReq->sql);
   }
   pReq->sql = NULL;
@@ -8194,7 +8169,7 @@ int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) {
 
     for (int32_t i = 0; i < pRsp->blockNum; i++) {
       int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i);
-      void   *data = taosArrayGetP(pRsp->blockData, i);
+      void *  data = taosArrayGetP(pRsp->blockData, i);
       if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1;
       if (pRsp->withSchema) {
         SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i);
@@ -8227,7 +8202,7 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) {
     }
 
     for (int32_t i = 0; i < pRsp->blockNum; i++) {
-      void    *data;
+      void *   data;
       uint64_t bLen;
       if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1;
       taosArrayPush(pRsp->blockData, &data);
@@ -8273,7 +8248,7 @@ int32_t tEncodeSTaosxRsp(SEncoder *pEncoder, const STaosxRsp *pRsp) {
   if (tEncodeI32(pEncoder, pRsp->createTableNum) < 0) return -1;
   if (pRsp->createTableNum) {
     for (int32_t i = 0; i < pRsp->createTableNum; i++) {
-      void   *createTableReq = taosArrayGetP(pRsp->createTableReq, i);
+      void *  createTableReq = taosArrayGetP(pRsp->createTableReq, i);
       int32_t createTableLen = *(int32_t *)taosArrayGet(pRsp->createTableLen, i);
       if (tEncodeBinary(pEncoder, createTableReq, createTableLen) < 0) return -1;
     }
@@ -8289,7 +8264,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) {
     pRsp->createTableLen = taosArrayInit(pRsp->createTableNum, sizeof(int32_t));
     pRsp->createTableReq = taosArrayInit(pRsp->createTableNum, sizeof(void *));
     for (int32_t i = 0; i < pRsp->createTableNum; i++) {
-      void    *pCreate = NULL;
+      void *   pCreate = NULL;
       uint64_t len;
       if (tDecodeBinaryAlloc(pDecoder, &pCreate, &len) < 0) return -1;
       int32_t l = (int32_t)len;
@@ -8591,7 +8566,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
       taosArrayDestroy(pTbData->aCol);
     } else {
       int32_t nRow = TARRAY_SIZE(pTbData->aRowP);
-      SRow  **rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
+      SRow ** rows = (SRow **)TARRAY_DATA(pTbData->aRowP);
 
       for (int32_t i = 0; i < nRow; ++i) {
         tRowDestroy(rows[i]);
@@ -8855,7 +8830,7 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq
     }
 
     for (int32_t i = 0; i < pReq->numOfCols; ++i) {
-      SSchema* pSchema = pReq->pSchema + i;
+      SSchema *pSchema = pReq->pSchema + i;
       if (tDecodeSSchema(&decoder, pSchema) < 0) return -1;
     }
   }
@@ -8866,7 +8841,7 @@ int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq
   return 0;
 }
 
-void    tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) {
+void tFreeSCMCreateViewReq(SCMCreateViewReq *pReq) {
   if (NULL == pReq) {
     return;
   }
@@ -8876,7 +8851,7 @@ void    tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) {
   taosMemoryFreeClear(pReq->pSchema);
 }
 
-int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq) {
+int32_t tSerializeSCMDropViewReq(void *buf, int32_t bufLen, const SCMDropViewReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
@@ -8894,7 +8869,7 @@ int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq
   return tlen;
 }
 
-int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq) {
+int32_t tDeserializeSCMDropViewReq(void *buf, int32_t bufLen, SCMDropViewReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -8910,7 +8885,7 @@ int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pR
   tDecoderClear(&decoder);
   return 0;
 }
-void    tFreeSCMDropViewReq(SCMDropViewReq* pReq) {
+void tFreeSCMDropViewReq(SCMDropViewReq *pReq) {
   if (NULL == pReq) {
     return;
   }
@@ -8918,7 +8893,7 @@ void    tFreeSCMDropViewReq(SCMDropViewReq* pReq) {
   taosMemoryFree(pReq->sql);
 }
 
-int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq) {
+int32_t tSerializeSViewMetaReq(void *buf, int32_t bufLen, const SViewMetaReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
@@ -8932,7 +8907,7 @@ int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pR
   return tlen;
 }
 
-int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq) {
+int32_t tDeserializeSViewMetaReq(void *buf, int32_t bufLen, SViewMetaReq *pReq) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -8964,8 +8939,7 @@ static int32_t tEncodeSViewMetaRsp(SEncoder *pEncoder, const SViewMetaRsp *pRsp)
   return 0;
 }
 
-
+int32_t tSerializeSViewMetaRsp(void *buf, int32_t bufLen, const SViewMetaRsp *pRsp) {
-int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
 
@@ -8998,7 +8972,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) {
     }
 
     for (int32_t i = 0; i < pRsp->numOfCols; ++i) {
-      SSchema* pSchema = pRsp->pSchema + i;
+      SSchema *pSchema = pRsp->pSchema + i;
       if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1;
     }
   }
@@ -9006,7 +8980,7 @@ static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) {
   return 0;
 }
 
-int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) {
+int32_t tDeserializeSViewMetaRsp(void *buf, int32_t bufLen, SViewMetaRsp *pRsp) {
   SDecoder decoder = {0};
   tDecoderInit(&decoder, buf, bufLen);
 
@@ -9019,7 +8993,7 @@ int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp)
   return 0;
 }
 
-void    tFreeSViewMetaRsp(SViewMetaRsp* pRsp) {
+void tFreeSViewMetaRsp(SViewMetaRsp *pRsp) {
   if (NULL == pRsp) {
     return;
   }
@@ -9064,7 +9038,7 @@ int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
   }
 
   for (int32_t i = 0; i < numOfMeta; ++i) {
-    SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp));
+    SViewMetaRsp *metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp));
     if (NULL == metaRsp) return -1;
     if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1;
     taosArrayPush(pRsp->pViewRsp, &metaRsp);
@@ -9086,7 +9060,3 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
 
   taosArrayDestroy(pRsp->pViewRsp);
 }
-
-
-
-

source/common/src/ttime.c

@@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
     i += 2;
   }
 
+  if (hour > 12 || hour < 0) {
+    return -1;
+  }
+
   // return error if there're illegal charaters after min(2 Digits)
   char* minStr = &str[i];
   if (minStr[1] != '\0' && minStr[2] != '\0') {
@@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
   }
 
   int64_t minute = strnatoi(&str[i], 2);
-  if (minute > 59) {
+  if (minute > 59 || (hour == 12 && minute > 0)) {
     return -1;
   }
 
@@ -646,7 +650,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
 
   /* get the basic numeric value */
   int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
-  if (timestamp < 0 || errno != 0) {
+  if ((timestamp == 0 && token[0] != '0') || errno != 0) {
     return -1;
   }
 

source/common/src/tvariant.c

@@ -19,6 +19,278 @@
 #include "ttokendef.h"
 #include "tvariant.h"
 
+int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
+  // skip head 0b
+  const char *p = z + 2;
+  int32_t     l = n - 2;
+
+  while (*p == '0' && l > 0) {
+    p++;
+    l--;
+  }
+  if (l > 64) {  // too big
+    return TSDB_CODE_FAILED;
+  }
+
+  uint64_t val = 0;
+  for (int32_t i = 0; i < l; i++) {
+    val = val << 1;
+    if (p[i] == '1') {
+      val |= 1;
+    } else if (p[i] != '0') {  // abnormal char
+      return TSDB_CODE_FAILED;
+    }
+  }
+  *value = val;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) {
+  // parse sign
+  bool has_sign = false;
+  if (z[0] == '-') {
+    *is_neg = true;
+    has_sign = true;
+  } else if (z[0] == '+') {
+    has_sign = true;
+  } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) {
+    return TSDB_CODE_FAILED;
+  }
+  if (has_sign) {
+    if (n < 2) {
+      return TSDB_CODE_FAILED;
+    }
+    z++;
+    n--;
+  }
+
+  errno = 0;
+  char  *endPtr = NULL;
+  bool parsed = false;
+  if (z[0] == '0' && n > 2) {
+    if (z[1] == 'b' || z[1] == 'B') {
+      // paring as binary
+      return parseBinaryUInteger(z, n, value);
+    }
+
+    if (z[1] == 'x' || z[1] == 'X') {
+      // parsing as hex
+      *value = taosStr2UInt64(z, &endPtr, 16);
+      parsed = true;
+    }
+  }
+
+  if (!parsed) {
+    // parsing as double
+    double val = taosStr2Double(z, &endPtr);
+    if (val > UINT64_MAX) {
+      return TSDB_CODE_FAILED;
+    }
+    *value = round(val);
+  }
+
+  if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) {
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  errno = 0;
+  char* endPtr = NULL;
+  *value = taosStr2Double(z, &endPtr);
+
+  if (errno == ERANGE || errno == EINVAL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  switch (type)
+  {
+    case TK_NK_INTEGER: {
+      *value = taosStr2Int64(z, &endPtr, 10);
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(z, &endPtr));
+      if (!IS_VALID_INT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+  default:
+    break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. try to parse as integer
+  *value = taosStr2Int64(z, &endPtr, 10);
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    // pure decimal part
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr+1 < end && endPtr[1] > '4') {
+        const char *p = z;
+        while (*p == ' ') {
+          p++;
+        }
+        if (*p == '-') {
+          if ( *value > INT64_MIN) {
+            (*value)--;
+          }
+        } else {
+          if ( *value < INT64_MAX) {
+            (*value)++;
+          }
+        }
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool     is_neg = false;
+  uint64_t uv = 0;
+  int32_t  code = parseSignAndUInteger(z, n, &is_neg, &uv);
+  if (code == TSDB_CODE_SUCCESS) {
+    // truncate into int64
+    if (is_neg) {
+      if (uv > 1ull + INT64_MAX) {
+        *value = INT64_MIN;
+        return TSDB_CODE_FAILED;
+      } else {
+        *value = -uv;
+      }
+    } else {
+      if (uv > INT64_MAX) {
+        *value = INT64_MAX;
+        return TSDB_CODE_FAILED;
+      }
+      *value = uv;
+    }
+  }
+
+  return code;
+}
+
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  const char *p = z;
+  while (*p == ' ') {
+    p++;
+  }
+  switch (type) {
+    case TK_NK_INTEGER: {
+      *value = taosStr2UInt64(p, &endPtr, 10);
+      if (*p == '-' && *value) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(p, &endPtr));
+      if (!IS_VALID_UINT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+    default:
+      break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. parse as integer
+  *value = taosStr2UInt64(p, &endPtr, 10);
+  if (*p == '-' && *value) {
+    return TSDB_CODE_FAILED;
+  }
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) {
+        (*value)++;
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool    is_neg = false;
+  int32_t code = parseSignAndUInteger(z, n, &is_neg, value);
+  if (is_neg) {
+    if (TSDB_CODE_SUCCESS == code && 0 == *value) {
+      return TSDB_CODE_SUCCESS;
+    }
+    return TSDB_CODE_FAILED;
+  }
+  return code;
+}
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   errno = 0;
   char *endPtr = NULL;
@@ -26,10 +298,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   *value = taosStr2Int64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
@@ -39,16 +311,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
   const char *p = z;
   while (*p == ' ') p++;
   if (*p == '-') {
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
   *value = taosStr2UInt64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
-
+  return TSDB_CODE_SUCCESS;
-  return 0;
 }
 
 /**
@@ -147,14 +418,13 @@ void taosVariantDestroy(SVariant *pVar) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
-
 }
 
 void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
 
   pDst->nType = pSrc->nType;
-  if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
+  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
       pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON ||
       pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
     int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
@@ -172,7 +442,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i = pSrc->i;
   }
-
 }
 
 int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {

source/common/test/commonTests.cpp

@@ -14,6 +14,7 @@
 #include "tdef.h"
 #include "tvariant.h"
 #include "ttime.h"
+#include "ttokendef.h"
 
 namespace {
 //
@@ -24,40 +25,177 @@ int main(int argc, char** argv) {
   return RUN_ALL_TESTS();
 }
 
-TEST(testCase, toInteger_test) {
+
+TEST(testCase, toUIntegerEx_test) {
+  uint64_t val = 0;
+
   char*    s = "123";
-  uint32_t type = 0;
+  int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
-
-  int64_t val = 0;
-
-  int32_t ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 123);
 
+  s = "1000u";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "0b110";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "2567.4787";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2567);
+
+  s = "1.869895343e4";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  s = "-1";
+  ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0b10010";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0x40";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-80.9999";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-5.2343544534e10";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  // INT64_MAX
   s = "9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 9223372036854775807);
 
-  s = "9323372036854775807";
+  // UINT64_MAX
-  ret = toInteger(s, strlen(s), 10, &val);
+  s = "18446744073709551615";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 9323372036854775807u);
+  ASSERT_EQ(val, 18446744073709551615u);
+
+  // out of range
+  s = "18446744073709551616";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "5.23e25";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toIntegerEx_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
+  s = "-1";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -1);
+
+  s = "1000u";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "-0x40";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -64);
+
+  s = "0b110";
+  ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "-0b10010";
+  ret = toIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -18);
+
+  s = "-80.9999";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -81);
+
+  s = "2567.8787";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2568);
+
+  s = "-5.2343544534e10";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -52343544534);
+
+  s = "1.869895343e4";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  // INT64_MAX
+  s = "9223372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 9223372036854775807LL);
+
+  s = "-9223372036854775808";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
 
   s = "-9323372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, -1);
 
+  // UINT64_MAX
+  s = "18446744073709551615";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toInteger_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
   s = "-1";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, -1);
 
-  s = "-9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
-  ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, -9223372036854775807);
-
   s = "1000u";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
@@ -77,13 +215,22 @@ TEST(testCase, toInteger_test) {
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 72);
 
-  // 18446744073709551615  UINT64_MAX
+  s = "9223372036854775807";
-  s = "18446744073709551615";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 18446744073709551615u);
+  ASSERT_EQ(val, 9223372036854775807);
 
-  s = "18446744073709551616";
+  s = "-9223372036854775808";
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807"; 
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-9323372036854775807";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
 }

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -28,6 +28,9 @@
     }                                \
   } while (0)
 
+extern int32_t streamTimerInit();
+extern void    streamTimerCleanUp();
+
 static SDnode globalDnode = {0};
 
 SDnode *dmInstance() { return &globalDnode; }
@@ -166,6 +169,7 @@ int32_t dmInit() {
 #if defined(USE_S3)
   if (s3Begin() != 0) return -1;
 #endif
+  if (streamTimerInit() != 0) return -1;
 
   dInfo("dnode env is initialized");
   return 0;
@@ -194,6 +198,8 @@ void dmCleanup() {
 #if defined(USE_S3)
   s3End();
 #endif
+  streamTimerCleanUp();
+
   dInfo("dnode env is cleaned up");
 
   taosCleanupCfg();

source/dnode/mnode/impl/inc/mndCluster.h

@@ -27,8 +27,6 @@ void    mndCleanupCluster(SMnode *pMnode);
 int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len);
 int64_t mndGetClusterId(SMnode *pMnode);
 int64_t mndGetClusterCreateTime(SMnode *pMnode);
-int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo);
-int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo);
 int64_t mndGetClusterUpTime(SMnode *pMnode);
 
 #ifdef __cplusplus

source/dnode/mnode/impl/inc/mndDef.h

@@ -192,8 +192,6 @@ typedef struct {
   int64_t createdTime;
   int64_t updateTime;
   int32_t upTime;
-  int64_t grantedTime;
-  int64_t connGrantedTime;
 } SClusterObj;
 
 typedef struct {

source/dnode/mnode/impl/src/mndCluster.c

@@ -19,7 +19,7 @@
 #include "mndTrans.h"
 
 #define CLUSTER_VER_NUMBE    1
-#define CLUSTER_RESERVE_SIZE 44
+#define CLUSTER_RESERVE_SIZE 60
 int64_t tsExpireTime = 0;
 
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
@@ -112,19 +112,6 @@ int64_t mndGetClusterCreateTime(SMnode *pMnode) {
   return createTime;
 }
 
-int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) {
-  void        *pIter = NULL;
-  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
-  if (pCluster != NULL) {
-    pInfo->grantedTime = pCluster->grantedTime;
-    pInfo->connGrantedTime = pCluster->connGrantedTime;
-    mndReleaseCluster(pMnode, pCluster, pIter);
-    return 0;
-  }
-
-  return -1;
-}
-
 static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) {
 #if 0
   int32_t upTime = taosGetTimestampSec() - pCluster->updateTime / 1000;
@@ -159,8 +146,6 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
   SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER)
   SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER)
   SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER)
-  SDB_SET_INT64(pRaw, dataPos, pCluster->grantedTime, _OVER)
-  SDB_SET_INT64(pRaw, dataPos, pCluster->connGrantedTime, _OVER)
   SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER)
 
   terrno = 0;
@@ -201,8 +186,6 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
   SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER)
   SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER)
   SDB_GET_INT32(pRaw, dataPos, &pCluster->upTime, _OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pCluster->grantedTime, _OVER);
-  SDB_GET_INT64(pRaw, dataPos, &pCluster->connGrantedTime, _OVER);
   SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER)
 
   terrno = 0;
@@ -235,8 +218,6 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj
   mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p, uptime from %d to %d", pOld->id, pOld,
          pNew, pOld->upTime, pNew->upTime);
   pOld->upTime = pNew->upTime;
-  pOld->grantedTime = pNew->grantedTime;
-  pOld->connGrantedTime = pNew->connGrantedTime;
   pOld->updateTime = taosGetTimestampMs();
   return 0;
 }
@@ -378,44 +359,3 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
   mndTransDrop(pTrans);
   return 0;
 }
-
-int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) {
-  SClusterObj  clusterObj = {0};
-  void        *pIter = NULL;
-  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
-  if (pCluster != NULL) {
-    if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime >= pInfo->connGrantedTime) {
-      mndReleaseCluster(pMnode, pCluster, pIter);
-      return 0;
-    }
-    memcpy(&clusterObj, pCluster, sizeof(SClusterObj));
-    if (pCluster->grantedTime < pInfo->grantedTime) clusterObj.grantedTime = pInfo->grantedTime;
-    if (pCluster->connGrantedTime < pInfo->connGrantedTime) clusterObj.connGrantedTime = pInfo->connGrantedTime;
-    mndReleaseCluster(pMnode, pCluster, pIter);
-  }
-
-  if (clusterObj.id <= 0) {
-    mError("can't get cluster info while update granted info");
-    return -1;
-  }
-
-  STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info");
-  if (pTrans == NULL) return -1;
-
-  SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj);
-  if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
-    mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
-    return -1;
-  }
-  (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
-
-  if (mndTransPrepare(pMnode, pTrans) != 0) {
-    mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
-    mndTransDrop(pTrans);
-    return -1;
-  }
-
-  mndTransDrop(pTrans);
-  return 0;
-}

source/dnode/mnode/impl/src/mndDnode.c

@@ -790,9 +790,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
           if (cfgAll) {  // alter all dnodes:
             if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
             if (failRecord) taosArrayPush(failRecord, &pDnode->id);
-            if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) {
+            if (0 == cfgAllErr) cfgAllErr = terrno;  // output 1st terrno.
-              cfgAllErr = terrno;  // output 1st or more specific error
-            }
           }
         } else {
           terrno = 0;  // no action for dup active code
@@ -808,9 +806,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
           if (cfgAll) {
             if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
             if (failRecord) taosArrayPush(failRecord, &pDnode->id);
-            if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) {
+            if (0 == cfgAllErr) cfgAllErr = terrno;
-              cfgAllErr = terrno;  // output 1st or more specific error
-            }
           }
         } else {
           terrno = 0;
@@ -1287,12 +1283,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
 
     strcpy(dcfgReq.config, "supportvnodes");
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-  } else if (strncasecmp(cfgReq.config, GRANT_ACTIVE_CODE, 10) == 0 ||
+  } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) {
-             strncasecmp(cfgReq.config, GRANT_C_ACTIVE_CODE, 11) == 0) {
-    if (cfgReq.dnodeId != -1) {
-      terrno = TSDB_CODE_INVALID_CFG;
-      goto _err_out;
-    }
     int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE;
     int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11;
     if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) {
@@ -1310,11 +1301,12 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
       goto _err_out;
     }
 
-    strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? GRANT_ACTIVE_CODE : GRANT_C_ACTIVE_CODE);
+    strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode");
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value);
 
-    if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) {
+    if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
       mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
+      terrno = TSDB_CODE_INVALID_CFG;
       goto _err_out;
     }
     tFreeSMCfgDnodeReq(&cfgReq);

source/dnode/mnode/impl/src/mndSnode.c

@@ -291,12 +291,17 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
-  pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
+//  pObj = mndAcquireSnode(pMnode, createReq.dnodeId);
-  if (pObj != NULL) {
+//  if (pObj != NULL) {
+//    terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
+//    goto _OVER;
+//  } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) {
+//    goto _OVER;
+//  }
+
+  if (sdbGetSize(pMnode->pSdb, SDB_SNODE) >= 1){
     terrno = TSDB_CODE_MND_SNODE_ALREADY_EXIST;
     goto _OVER;
-  } else if (terrno != TSDB_CODE_MND_SNODE_NOT_EXIST) {
-    goto _OVER;
   }
 
   pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
@@ -314,7 +319,7 @@ _OVER:
     return -1;
   }
 
-  mndReleaseSnode(pMnode, pObj);
+//  mndReleaseSnode(pMnode, pObj);
   mndReleaseDnode(pMnode, pDnode);
   tFreeSMCreateQnodeReq(&createReq);
   return code;

source/dnode/mnode/impl/src/mndStream.c

@@ -363,35 +363,6 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
   return 0;
 }
 
-static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64_t watermark, char **pStr) {
-  if (NULL == ast) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  SNode * pAst = NULL;
-  int32_t code = nodesStringToNode(ast, &pAst);
-
-  SQueryPlan *pPlan = NULL;
-  if (TSDB_CODE_SUCCESS == code) {
-    SPlanContext cxt = {
-        .pAstRoot = pAst,
-        .topicQuery = false,
-        .streamQuery = true,
-        .triggerType = (triggerType == STREAM_TRIGGER_MAX_DELAY) ? STREAM_TRIGGER_WINDOW_CLOSE : triggerType,
-        .watermark = watermark,
-    };
-    code = qCreateQueryPlan(&cxt, &pPlan, NULL);
-  }
-
-  if (TSDB_CODE_SUCCESS == code) {
-    code = nodesNodeToString((SNode *)pPlan, false, pStr, NULL);
-  }
-  nodesDestroyNode(pAst);
-  nodesDestroyNode((SNode *)pPlan);
-  terrno = code;
-  return code;
-}
-
 static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) {
   SNode *     pAst = NULL;
   SQueryPlan *pPlan = NULL;
@@ -734,10 +705,19 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask
 
   STransAction action = {0};
   SEpSet  epset = {0};
-  if (pTask->info.nodeId == SNODE_HANDLE) {
+  if(pTask->info.nodeId == SNODE_HANDLE){
-    SSnodeObj *pObj = mndAcquireSnode(pMnode, pTask->info.nodeId);
+    SSnodeObj *pObj = NULL;
+    void  *pIter = NULL;
+    while (1) {
+      pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
+      if (pIter == NULL) {
+        break;
+      }
+
       addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port);
-  } else {
+      sdbRelease(pMnode->pSdb, pObj);
+    }
+  }else{
     SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
     epset = mndGetVgroupEpset(pMnode, pVgObj);
     mndReleaseVgroup(pMnode, pVgObj);

source/dnode/mnode/impl/src/mndSubscribe.c

@@ -161,6 +161,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj
 static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub,
                                         const SMqRebOutputVg *pRebVg, SSubplan* pPlan) {
   if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
+    if(pRebVg->oldConsumerId == -1) return 0;   //drop stream, no consumer, while split vnode,all consumerId is -1
     terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
     return -1;
   }

source/dnode/snode/src/snodeInitApi.c

@@ -80,6 +80,7 @@ void initStateStoreAPI(SStateStore* pStore) {
   pStore->updateInfoDestroy = updateInfoDestroy;
   pStore->windowSBfDelete = windowSBfDelete;
   pStore->windowSBfAdd = windowSBfAdd;
+  pStore->isIncrementalTimeStamp = isIncrementalTimeStamp;
 
   pStore->updateInfoInitP = updateInfoInitP;
   pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;

source/dnode/vnode/inc/vnode.h

@@ -169,7 +169,9 @@ void        *tsdbGetIdx2(SMeta *pMeta);
 void        *tsdbGetIvtIdx2(SMeta *pMeta);
 uint64_t     tsdbGetReaderMaxVersion2(STsdbReader *pReader);
 void         tsdbReaderSetCloseFlag(STsdbReader *pReader);
-//======================================================================================================================
+int64_t      tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
+void         tsdbSetFilesetDelimited(STsdbReader* pReader);
+void         tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 
 int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,

source/dnode/vnode/src/inc/tsdb.h

@@ -377,6 +377,7 @@ struct STsdb {
   SVnode              *pVnode;
   STsdbKeepCfg         keepCfg;
   TdThreadMutex        mutex;
+  bool                 bgTaskDisabled;
   SMemTable           *mem;
   SMemTable           *imem;
   STsdbFS              fs;  // old

source/dnode/vnode/src/inc/vnodeInt.h

@@ -223,8 +223,6 @@ int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid
 int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg);
 
 // tq
-int     tqInit();
-void    tqCleanUp();
 STQ*    tqOpen(const char* path, SVnode* pVnode);
 void    tqNotifyClose(STQ*);
 void    tqClose(STQ*);

source/dnode/vnode/src/tq/tq.c

@@ -17,12 +17,6 @@
 #include "vnd.h"
 #include "tqCommon.h"
 
-typedef struct {
-  int8_t inited;
-} STqMgmt;
-
-static STqMgmt tqMgmt = {0};
-
 // 0: not init
 // 1: already inited
 // 2: wait to be inited or cleaup
@@ -32,36 +26,6 @@ static FORCE_INLINE bool tqIsHandleExec(STqHandle* pHandle) { return TMQ_HANDLE_
 static FORCE_INLINE void tqSetHandleExec(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_EXEC; }
 static FORCE_INLINE void tqSetHandleIdle(STqHandle* pHandle) { pHandle->status = TMQ_HANDLE_STATUS_IDLE; }
 
-int32_t tqInit() {
-  int8_t old;
-  while (1) {
-    old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 2);
-    if (old != 2) break;
-  }
-
-  if (old == 0) {
-    if (streamInit() < 0) {
-      return -1;
-    }
-    atomic_store_8(&tqMgmt.inited, 1);
-  }
-
-  return 0;
-}
-
-void tqCleanUp() {
-  int8_t old;
-  while (1) {
-    old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 2);
-    if (old != 2) break;
-  }
-
-  if (old == 1) {
-    streamCleanUp();
-    atomic_store_8(&tqMgmt.inited, 0);
-  }
-}
-
 void tqDestroyTqHandle(void* data) {
   STqHandle* pData = (STqHandle*)data;
   qDestroyTask(pData->execHandle.task);
@@ -337,7 +301,7 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) {
 
     while (pIter) {
       STqHandle* pHandle = *(STqHandle**)pIter;
-      tqInfo("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId);
+      tqDebug("vgId:%d start set submit for pHandle:%p, consumer:0x%" PRIx64, vgId, pHandle, pHandle->consumerId);
 
       if (ASSERT(pHandle->msg != NULL)) {
         tqError("pHandle->msg should not be null");

source/dnode/vnode/src/tq/tqPush.c

@@ -72,7 +72,7 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) {
   memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen);
   pHandle->msg->contLen = pMsg->contLen;
   int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES);
-  tqInfo("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret,
+  tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64 ", register to pHandle:%p, pCont:%p, len:%d", vgId, ret,
           pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen);
   return 0;
 }

source/dnode/vnode/src/tq/tqUtil.c

@@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
     taosWUnLockLatch(&pTq->lock);
   }
 
+  dataRsp.reqOffset = *pOffset;   // reqOffset represents the current date offset, may be changed if wal not exists
   code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
 
 end : {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1797,8 +1797,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
       }
 
       if (record.version <= pReader->info.verRange.maxVer) {
-        tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
+        /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
-                  TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
+          TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/
 
         SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
         taosArrayPush(pInfo->pTombData, &delData);

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -22,17 +22,32 @@
 
 #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
 
+static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
+  char *buf = taosMemoryCalloc(1, pCol->info.bytes);
+  SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
+  pRes->bytes = 0;
+  pRes->hasResult = true;
+  pRes->isNull = true;
+  varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE);
+  colDataSetVal(pCol, row, buf, false);
+  taosMemoryFree(buf);
+}
+
 static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds,
                           const int32_t* dstSlotIds, void** pRes, const char* idStr) {
   int32_t numOfRows = pBlock->info.rows;
   // bool    allNullRow = true;
 
   if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
-    uint64_t ts = 0;
+    uint64_t ts = TSKEY_MIN;
-    SFirstLastRes* p;
+    SFirstLastRes* p = NULL;
-    col_id_t colId;
+    col_id_t colId = -1;
     for (int32_t i = 0; i < pReader->numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
+      if (slotIds[i] == -1) {
+        setFirstLastResColToNull(pColInfoData, numOfRows);
+        continue;
+      }
       int32_t   slotId = slotIds[i];
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       colId = pColVal->colVal.cid;
@@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
     for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
       SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
       if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
+        if (ts == TSKEY_MIN) {
+          colDataSetNULL(pCol, numOfRows);
+        } else {
           colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        }
         continue;
-      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) {
+      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) {
-        if (!p->isNull) {
+        if (p && !p->isNull) {
           colDataSetVal(pCol, numOfRows, p->buf, false);
         } else {
           colDataSetNULL(pCol, numOfRows);
@@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
 
       int32_t   slotId = slotIds[i];
+      if (slotId == -1) {
+        colDataSetNULL(pColInfoData, numOfRows);
+        continue;
+      }
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       SColVal*  pVal = &pColVal->colVal;
 
@@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   }
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
-    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE);
+    int32_t bytes;
+    if (slotIds[j] == -1)
+      bytes = 1;
+    else
+      bytes = pr->pSchema->columns[slotIds[j]].bytes;
+
+    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE);
     SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
     p->ts = INT64_MIN;
   }
@@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
     for (int32_t i = 0; i < pr->numOfCols; ++i) {
       int32_t          slotId = slotIds[i];
+      if (slotId == -1) {
+        SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL};
+        taosArrayPush(pLastCols, &p);
+        continue;
+      }
       struct STColumn* pCol = &pr->pSchema->columns[slotId];
       SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
 
@@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         bool    hasNotNullRow = true;
         int64_t singleTableLastTs = INT64_MAX;
         for (int32_t k = 0; k < pr->numOfCols; ++k) {
+          if (slotIds[k] == -1) continue;
           SLastCol* p = taosArrayGet(pLastCols, k);
           SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
 

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -745,18 +745,28 @@ _exit:
   return code;
 }
 
-int32_t tsdbFSCancelAllBgTask(STFileSystem *fs) {
+static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block);
+
+int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
+  STFileSystem *fs = pTsdb->pFS;
   TARRAY2(int64_t) channelArr = {0};
 
-  // collect all open channels
   taosThreadMutexLock(&fs->tsdb->mutex);
+
+  // disable
+  pTsdb->bgTaskDisabled = true;
+
+  // collect channel
   STFileSet *fset;
   TARRAY2_FOREACH(fs->fSetArr, fset) {
     if (VNODE_ASYNC_VALID_CHANNEL_ID(fset->bgTaskChannel)) {
       TARRAY2_APPEND(&channelArr, fset->bgTaskChannel);
       fset->bgTaskChannel = 0;
     }
+    fset->mergeScheduled = false;
+    tsdbFSSetBlockCommit(fset, false);
   }
+
   taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   // destroy all channels
@@ -766,10 +776,17 @@ int32_t tsdbFSCancelAllBgTask(STFileSystem *fs) {
   return 0;
 }
 
+int32_t tsdbEnableBgTask(STsdb *pTsdb) {
+  taosThreadMutexLock(&pTsdb->mutex);
+  pTsdb->bgTaskDisabled = false;
+  taosThreadMutexUnlock(&pTsdb->mutex);
+  return 0;
+}
+
 int32_t tsdbCloseFS(STFileSystem **fs) {
   if (fs[0] == NULL) return 0;
 
-  tsdbFSCancelAllBgTask(*fs);
+  tsdbDisableAndCancelAllBgTask((*fs)->tsdb);
   close_file_system(fs[0]);
   destroy_fs(fs);
   return 0;
@@ -857,7 +874,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
 
   // schedule merge
   int32_t sttTrigger = fs->tsdb->pVnode->config.sttTrigger;
-  if (sttTrigger > 1) {
+  if (sttTrigger > 1 && !fs->tsdb->bgTaskDisabled) {
     STFileSet *fset;
     TARRAY2_FOREACH_REVERSE(fs->fSetArr, fset) {
       if (TARRAY2_SIZE(fset->lvlArr) == 0) {
@@ -873,7 +890,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
 
       bool    skipMerge = false;
       int32_t numFile = TARRAY2_SIZE(lvl->fobjArr);
-      if (numFile >= sttTrigger) {
+      if (numFile >= sttTrigger && (!fset->mergeScheduled)) {
         // launch merge
         {
           extern int8_t  tsS3Enabled;
@@ -917,6 +934,7 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) {
           code = vnodeAsyncC(vnodeAsyncHandle[1], fset->bgTaskChannel, EVA_PRIORITY_HIGH, tsdbMerge, taosMemoryFree,
                              arg, NULL);
           TSDB_CHECK_CODE(code, lino, _exit);
+          fset->mergeScheduled = true;
         }
       }
 

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -453,6 +453,7 @@ int32_t tsdbTFileSetInit(int32_t fid, STFileSet **fset) {
 
   // background task queue
   fset[0]->bgTaskChannel = 0;
+  fset[0]->mergeScheduled = false;
 
   // block commit variables
   taosThreadCondInit(&fset[0]->canCommit, NULL);

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -93,6 +93,7 @@ struct STFileSet {
 
   // background task channel
   int64_t bgTaskChannel;
+  bool    mergeScheduled;
 
   // block commit variables
   TdThreadCond canCommit;

source/dnode/vnode/src/tsdb/tsdbMerge.c

@@ -514,6 +514,8 @@ static int32_t tsdbMergeGetFSet(SMerger *merger) {
     return 0;
   }
 
+  fset->mergeScheduled = false;
+
   int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset);
   if (code) {
     taosThreadMutexUnlock(&merger->tsdb->mutex);

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -70,6 +70,8 @@ static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFil
 static bool          hasDataInSttBlock(SSttBlockReader* pSttBlockReader);
 static void          initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter);
 static int32_t       getInitialDelIndex(const SArray* pDelSkyline, int32_t order);
+static void          resetTableListIndex(SReaderStatus* pStatus);
+static void          getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey);
 static void          updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);
 
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
@@ -254,6 +256,7 @@ static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bo
 
     tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey,
               pReader->info.window.ekey, pReader->idStr);
+
     *hasNext = true;
     return TSDB_CODE_SUCCESS;
   }
@@ -432,6 +435,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void
     goto _end;
   }
 
+  pReader->bFilesetDelimited = false;
+
   tsdbInitReaderLock(pReader);
   tsem_init(&pReader->resumeAfterSuspend, 0, 0);
 
@@ -2531,6 +2536,41 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader)
   }
 }
 
+static void prepareDurationForNextFileSet(STsdbReader* pReader) {
+  if (pReader->status.bProcMemFirstFileset) {
+    pReader->status.prevFilesetStartKey = INT64_MIN;
+    pReader->status.prevFilesetEndKey = INT64_MAX;
+    pReader->status.bProcMemFirstFileset = false;
+  }
+
+  int32_t fid = pReader->status.pCurrentFileset->fid;
+  STimeWindow winFid = {0};
+  tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey);
+
+  if (ASCENDING_TRAVERSE(pReader->info.order)) {
+    pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey ||
+                                            (winFid.skey-1) < pReader->status.memTableMinKey);
+  } else {
+    pReader->status.bProcMemPreFileset = !( pReader->status.memTableMaxKey < (winFid.ekey+1) ||
+                                            pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey);
+  }
+
+  if (pReader->status.bProcMemPreFileset) {
+    resetTableListIndex(&pReader->status);
+  }
+
+  if (!pReader->status.bProcMemPreFileset) {
+    if (pReader->notifyFn) {
+      STsdReaderNotifyInfo info = {0};
+      info.duration.filesetId = fid;
+      pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
+    }
+  }
+
+  pReader->status.prevFilesetStartKey = winFid.skey;
+  pReader->status.prevFilesetEndKey = winFid.ekey;
+}
+
 static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) {
   SReaderStatus* pStatus = &pReader->status;
   pBlockNum->numOfBlocks = 0;
@@ -2572,6 +2612,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr
       }
 
       if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) {
+        if (pReader->bFilesetDelimited) {
+          prepareDurationForNextFileSet(pReader);
+        }
         break;
       }
     }
@@ -2849,13 +2892,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) {
       if (!hasNexTable) {
         return TSDB_CODE_SUCCESS;
       }
-      pBlockScanInfo = pStatus->pTableIter;
+      continue;
     }
 
     initMemDataIterator(*pBlockScanInfo, pReader);
     initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);
 
-    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
     int32_t code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -3685,7 +3727,7 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
     }
 
     if (row.type == TSDBROW_ROW_FMT) {
-      int64_t ts = row.pTSRow->ts;;
+      int64_t ts = row.pTSRow->ts;
       code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
 
       if (freeTSRow) {
@@ -3792,6 +3834,11 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) {
   SReaderStatus*  pStatus = &pReader->status;
   SDataBlockIter* pBlockIter = &pStatus->blockIter;
 
+  if (pReader->bFilesetDelimited) {
+    getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey);
+    pReader->status.bProcMemFirstFileset = true;
+  }
+
   initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);
   resetDataBlockIterator(&pStatus->blockIter, pReader->info.order);
 
@@ -4094,6 +4141,10 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
 
   tsdbUntakeReadSnap2(pReader, pReader->pReadSnap, false);
   pReader->pReadSnap = NULL;
+  if (pReader->bFilesetDelimited) {
+    pReader->status.memTableMinKey = INT64_MAX;
+    pReader->status.memTableMaxKey = INT64_MIN;
+  }
   pReader->flag = READER_STATUS_SUSPEND;
 
 #if SUSPEND_RESUME_TEST
@@ -4179,6 +4230,72 @@ _err:
   return code;
 }
 
+static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
+  SReaderStatus* pStatus = &pReader->status;
+  int32_t code = TSDB_CODE_SUCCESS;
+  SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock;
+
+  if (pStatus->loadFromFile) {
+    if (pStatus->bProcMemPreFileset) {
+      int32_t fid = pReader->status.pCurrentFileset->fid;
+      STimeWindow win = {0};
+      tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);
+
+      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey;
+      code = buildBlockFromBufferSequentially(pReader, endKey);
+      if (code != TSDB_CODE_SUCCESS || pBlock->info.rows > 0) {
+        return code;
+      } else {
+        pStatus->bProcMemPreFileset = false;
+        if (pReader->notifyFn) {
+          STsdReaderNotifyInfo info = {0};
+          info.duration.filesetId = fid;
+          pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
+        }
+        resetTableListIndex(pStatus);
+      }
+    }
+
+    code = buildBlockFromFiles(pReader);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    if (pBlock->info.rows <= 0) {
+      resetTableListIndex(&pReader->status);
+      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
+      code = buildBlockFromBufferSequentially(pReader, endKey);
+    }
+  } else {  // no data in files, let's try the buffer
+    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
+    code = buildBlockFromBufferSequentially(pReader, endKey);
+  }
+  return code;
+}
+
+static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) {
+  SReaderStatus* pStatus = &pReader->status;
+  int32_t code = TSDB_CODE_SUCCESS;
+  SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock;
+
+  if (pStatus->loadFromFile) {
+    code = buildBlockFromFiles(pReader);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    if (pBlock->info.rows <= 0) {
+      resetTableListIndex(&pReader->status);
+      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
+      code = buildBlockFromBufferSequentially(pReader, endKey);
+    }
+  } else {  // no data in files, let's try the buffer
+    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
+    code = buildBlockFromBufferSequentially(pReader, endKey);
+  }
+  return code;
+}
+
 static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -4193,18 +4310,10 @@ static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
     return code;
   }
 
-  if (pStatus->loadFromFile) {
+  if (!pReader->bFilesetDelimited) {
-    code = buildBlockFromFiles(pReader);
+    code = doTsdbNextDataBlockFilesFirst(pReader);
-    if (code != TSDB_CODE_SUCCESS) {
+  } else {
-      return code;
+    code = doTsdbNextDataBlockFilesetDelimited(pReader);
-    }
-
-    if (pBlock->info.rows <= 0) {
-      resetTableListIndex(&pReader->status);
-      code = buildBlockFromBufferSequentially(pReader);
-    }
-  } else {  // no data in files, let's try the buffer
-    code = buildBlockFromBufferSequentially(pReader);
   }
 
   *hasNext = pBlock->info.rows > 0;
@@ -4692,6 +4801,54 @@ int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pT
   return code;
 }
 
+static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  int64_t rows = 0;
+
+  SReaderStatus* pStatus = &pReader->status;
+
+  int32_t iter = 0;
+  int64_t maxKey = INT64_MIN;
+  int64_t minKey = INT64_MAX;
+
+  void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter);
+  while (pHashIter!= NULL) {
+    STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter;
+
+    STbData* d = NULL;
+    if (pReader->pReadSnap->pMem != NULL) {
+      d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid);
+      if (d != NULL) {
+        if (d->maxKey > maxKey) {
+          maxKey = d->maxKey;
+        }
+        if (d->minKey < minKey) {
+          minKey = d->minKey;
+        }
+      }
+    }
+
+    STbData* di = NULL;
+    if (pReader->pReadSnap->pIMem != NULL) {
+      di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid);
+      if (di != NULL) {
+        if (di->maxKey > maxKey) {
+          maxKey = di->maxKey;
+        }
+        if (di->minKey < minKey) {
+          minKey = di->minKey;
+        }
+      }
+    }
+
+    // current table is exhausted, let's try the next table
+    pHashIter = tSimpleHashIterate(pStatus->pTableMap, pHashIter, &iter);
+  }
+
+  *pMaxKey = maxKey;
+  *pMinKey = minKey;
+}
+
 int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) {
   int32_t code = TSDB_CODE_SUCCESS;
   int64_t rows = 0;
@@ -4879,3 +5036,12 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) {
 
 void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/
 }
+
+void tsdbSetFilesetDelimited(STsdbReader* pReader) {
+  pReader->bFilesetDelimited = true;
+}
+
+void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) {
+  pReader->notifyFn = notifyFn;
+  pReader->notifyParam = param;
+}

source/dnode/vnode/src/tsdb/tsdbReadUtil.h

@@ -22,6 +22,7 @@ extern "C" {
 
 #include "tsdbDataFileRW.h"
 #include "tsdbUtil2.h"
+#include "storageapi.h"
 
 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC)
 
@@ -208,6 +209,12 @@ typedef struct SReaderStatus {
   SArray*               pLDataIterArray;
   SRowMerger            merger;
   SColumnInfoData*      pPrimaryTsCol;  // primary time stamp output col info data
+  bool                  bProcMemPreFileset;
+  int64_t               memTableMaxKey;
+  int64_t               memTableMinKey;
+  int64_t               prevFilesetStartKey;
+  int64_t               prevFilesetEndKey;
+  bool                  bProcMemFirstFileset;
 } SReaderStatus;
 
 struct STsdbReader {
@@ -231,6 +238,9 @@ struct STsdbReader {
   SBlockInfoBuf      blockInfoBuf;
   EContentData       step;
   STsdbReader*       innerReader[2];
+  bool                 bFilesetDelimited;   // duration by duration output
+  TsdReaderNotifyCbFn  notifyFn;
+  void*              notifyParam;
 };
 
 typedef struct SBrinRecordIter {

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -430,6 +430,11 @@ int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) {
 
   taosThreadMutexLock(&tsdb->mutex);
 
+  if (tsdb->bgTaskDisabled) {
+    taosThreadMutexUnlock(&tsdb->mutex);
+    return 0;
+  }
+
   STFileSet *fset;
   TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
     code = tsdbTFileSetOpenChannel(fset);

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -278,6 +278,15 @@ _exit:
   return code;
 }
 
+static int64_t tBlockDataSize(SBlockData* pBlockData) {
+  int64_t nData = 0;
+  for (int32_t iCol = 0; iCol < pBlockData->nColData; iCol++) {
+    SColData* pColData = tBlockDataGetColDataByIdx(pBlockData, iCol);
+    nData += pColData->nData;
+  }
+  return nData;
+}
+
 static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** data) {
   int32_t   code = 0;
   int32_t   lino = 0;
@@ -320,10 +329,13 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat
     code = tsdbIterMergerNext(reader->dataIterMerger);
     TSDB_CHECK_CODE(code, lino, _exit);
 
-    if (reader->blockData->nRow >= 81920) {
+    if (!(reader->blockData->nRow % 16)) {
+      int64_t nData = tBlockDataSize(reader->blockData);
+      if (nData >= 1 * 1024 * 1024) {
         break;
       }
     }
+  }
 
   if (reader->blockData->nRow > 0) {
     ASSERT(reader->blockData->suid || reader->blockData->uid);
@@ -1591,6 +1603,3 @@ _out:
 
   return code;
 }
-
-extern int32_t tsdbFSCancelAllBgTask(STFileSystem* fs);
-int32_t        tsdbCancelAllBgTask(STsdb* tsdb) { return tsdbFSCancelAllBgTask(tsdb->pFS); }

source/dnode/vnode/src/vnd/vnodeInitApi.c

@@ -60,6 +60,9 @@ void initTsdbReaderAPI(TsdReader* pReader) {
 
   pReader->tsdSetQueryTableList = tsdbSetTableList2;
   pReader->tsdSetReaderTaskId = (void (*)(void*, const char*))tsdbReaderSetId2;
+
+  pReader->tsdSetFilesetDelimited = (void (*)(void*))tsdbSetFilesetDelimited;
+  pReader->tsdSetSetNotifyCb = (void (*)(void*, TsdReaderNotifyCbFn, void*))tsdbReaderSetNotifyCb;
 }
 
 void initMetadataAPI(SStoreMeta* pMeta) {
@@ -189,6 +192,7 @@ void initStateStoreAPI(SStateStore* pStore) {
   pStore->updateInfoDestroy = updateInfoDestroy;
   pStore->windowSBfDelete = windowSBfDelete;
   pStore->windowSBfAdd = windowSBfAdd;
+  pStore->isIncrementalTimeStamp = isIncrementalTimeStamp;
 
   pStore->updateInfoInitP = updateInfoInitP;
   pStore->updateInfoAddCloseWindowSBF = updateInfoAddCloseWindowSBF;

source/dnode/vnode/src/vnd/vnodeModule.c

@@ -39,12 +39,6 @@ int vnodeInit(int nthreads) {
   if (walInit() < 0) {
     return -1;
   }
-  if (tqInit() < 0) {
-    return -1;
-  }
-  if (s3Init() < 0) {
-    return -1;
-  }
 
   return 0;
 }
@@ -58,7 +52,5 @@ void vnodeCleanup() {
   vnodeAsyncDestroy(&vnodeAsyncHandle[1]);
 
   walCleanUp();
-  tqCleanUp();
   smaCleanUp();
-  s3CleanUp();
 }

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -519,7 +519,21 @@ _out:
   return code;
 }
 
-extern int32_t tsdbCancelAllBgTask(STsdb *tsdb);
+extern int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb);
+extern int32_t tsdbEnableBgTask(STsdb *pTsdb);
+
+static int32_t vnodeCancelAndDisableAllBgTask(SVnode *pVnode) {
+  tsdbDisableAndCancelAllBgTask(pVnode->pTsdb);
+  vnodeSyncCommit(pVnode);
+  vnodeAChannelDestroy(vnodeAsyncHandle[0], pVnode->commitChannel, true);
+  return 0;
+}
+
+static int32_t vnodeEnableBgTask(SVnode *pVnode) {
+  tsdbEnableBgTask(pVnode->pTsdb);
+  vnodeAChannelInit(vnodeAsyncHandle[0], &pVnode->commitChannel);
+  return 0;
+}
 
 int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter **ppWriter) {
   int32_t       code = 0;
@@ -527,9 +541,8 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
   int64_t       sver = pParam->start;
   int64_t       ever = pParam->end;
 
-  // commit memory data
+  // cancel and disable all bg task
-  vnodeSyncCommit(pVnode);
+  vnodeCancelAndDisableAllBgTask(pVnode);
-  tsdbCancelAllBgTask(pVnode->pTsdb);
 
   // alloc
   pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter));
@@ -659,6 +672,7 @@ _exit:
     vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback);
     taosMemoryFree(pWriter);
   }
+  vnodeEnableBgTask(pVnode);
   return code;
 }
 

source/libs/executor/inc/executorInt.h

@@ -270,6 +270,7 @@ typedef struct STableScanInfo {
   bool            hasGroupByTag;
   bool            countOnly;
   //  TsdReader    readerAPI;
+  bool            filesetDelimited;
 } STableScanInfo;
 
 typedef struct STableMergeScanInfo {
@@ -297,6 +298,9 @@ typedef struct STableMergeScanInfo {
   SHashObj*        mSkipTables;
   int64_t          mergeLimit;
   SSortExecInfo   sortExecInfo;
+  bool             bNewFileset;
+  bool             bOnlyRetrieveBlock;
+  bool             filesetDelimited;
 } STableMergeScanInfo;
 
 typedef struct STagScanFilterContext {
@@ -350,6 +354,7 @@ typedef struct SStreamAggSupporter {
   SStateStore         stateStore;
   STimeWindow         winRange;
   SStorageAPI*        pSessionAPI;
+  struct SUpdateInfo* pUpdateInfo;
 } SStreamAggSupporter;
 
 typedef struct SWindowSupporter {
@@ -534,6 +539,7 @@ typedef struct SStreamIntervalOperatorInfo {
   SOpCheckPointInfo   checkPointInfo;
   bool                reCkBlock;
   SSDataBlock*        pCheckpointRes;
+  struct SUpdateInfo* pUpdateInfo;
 } SStreamIntervalOperatorInfo;
 
 typedef struct SDataGroupInfo {
@@ -830,6 +836,7 @@ void     compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeW
                            SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap);
 int32_t  releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI);
 void     resetWinRange(STimeWindow* winRange);
+bool     checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts);
 
 int32_t encodeSSessionKey(void** buf, SSessionKey* key);
 void*   decodeSSessionKey(void* buf, SSessionKey* key);

source/libs/executor/src/cachescanoperator.c

@@ -352,6 +352,7 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i);
+    bool found = false;
     for (int32_t j = 0; j < pWrapper->nCols; ++j) {
       /*      if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
         (*pSlotIds)[pColMatch->dstSlotId] = -1;
@@ -361,9 +362,14 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
       if (pColMatch->colId == pWrapper->pSchema[j].colId) {
         (*pSlotIds)[i] = j;
         (*pDstSlotIds)[i] = pColMatch->dstSlotId;
+        found = true;
         break;
       }
     }
+    if (!found) {
+      (*pSlotIds)[i] = -1;
+      (*pDstSlotIds)[i] = pColMatch->dstSlotId;
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/executor/src/dataInserter.c

@@ -189,7 +189,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
   }
 
   int64_t lastTs = TSKEY_MIN;
-  bool    ignoreRow = false;
+  bool    updateLastRow = false;
   bool    disorderTs = false;
 
   for (int32_t j = 0; j < rows; ++j) {  // iterate by row
@@ -249,7 +249,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
             } else {
               if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
                 if (*(int64_t*)var == lastTs) {
-                  ignoreRow = true;
+                  updateLastRow = true;
                 } else if (*(int64_t*)var < lastTs) {
                   disorderTs = true;
                 } else {
@@ -269,15 +269,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
           }
           break;
       }
-
-      if (ignoreRow) {
-        break;
-      }
-    }
-
-    if (ignoreRow) {
-      ignoreRow = false;
-      continue;
     }
 
     SRow* pRow = NULL;
@@ -285,7 +276,14 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
       tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
       goto _end;
     }
+    if (updateLastRow) {
+      updateLastRow = false;
+      SRow** lastRow = taosArrayPop(tbData.aRowP);
+      tRowDestroy(*lastRow);
       taosArrayPush(tbData.aRowP, &pRow);
+    } else {
+      taosArrayPush(tbData.aRowP, &pRow);
+    }
   }
 
   if (disorderTs) {

source/libs/executor/src/scanoperator.c

@@ -893,7 +893,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
     if (code != TSDB_CODE_SUCCESS) {
       T_LONG_JMP(pTaskInfo->env, code);
     }
-  
+    if (pInfo->filesetDelimited) {
+      pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader);
+    }
     if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) {
       pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity;
     }
@@ -1084,6 +1086,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
     pInfo->countOnly = true;
   }
 
+  pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
+
   taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
   pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo,
                                          optrDefaultBufFn, getTableScannerExecInfo, optrDefaultGetNextExtFn, NULL);
@@ -1618,6 +1622,15 @@ void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKE
   pBlock->info.rows++;
 }
 
+bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts) {
+  bool isExpired = false;
+  bool isInc = pAPI->isIncrementalTimeStamp(pUpdateInfo, tableId, ts);
+  if (!isInc) {
+    isExpired = isOverdue(ts, pTwSup);
+  }
+  return isExpired;
+}
+
 static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
   if (out) {
     blockDataCleanup(pInfo->pUpdateDataRes);
@@ -3213,6 +3226,7 @@ static int32_t tableMergeScanDoSkipTable(STableMergeScanInfo* pInfo, SSDataBlock
     tSimpleHashPut(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &nRows, sizeof(nRows));
   } else {
     *(int64_t*)pNum = *(int64_t*)pNum + pBlock->info.rows;
+    nRows = *(int64_t*)pNum;
   }
 
   if (nRows >= pInfo->mergeLimit) {
@@ -3241,6 +3255,7 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
 
   STsdbReader* reader = pInfo->base.dataReader;
   while (true) {
+    if (!pInfo->bOnlyRetrieveBlock) {
       code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext);
       if (code != 0) {
         pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
@@ -3248,16 +3263,20 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
         T_LONG_JMP(pTaskInfo->env, code);
       }
 
-    if (!hasNext) {
+      if (!hasNext || isTaskKilled(pTaskInfo)) {
-      break;
+        pInfo->bNewFileset = false;
-    }
-
         if (isTaskKilled(pTaskInfo)) {
           qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo));
           pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
+        }
         break;
       }
 
+      if (pInfo->bNewFileset) {
+        pInfo->bOnlyRetrieveBlock = true;
+        return NULL;
+      }
+    }
     // process this data block based on the probabilities
     bool processThisBlock = processBlockWithProbability(&pInfo->sample);
     if (!processThisBlock) {
@@ -3266,7 +3285,9 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
 
     uint32_t status = 0;
     code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
-    //    code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status);
+    if (pInfo->bOnlyRetrieveBlock) {
+      pInfo->bOnlyRetrieveBlock = false;
+    }
     if (code != TSDB_CODE_SUCCESS) {
       qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo));
       T_LONG_JMP(pTaskInfo->env, code);
@@ -3325,6 +3346,60 @@ int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond*
   return 0;
 }
 
+void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) {
+  STableMergeScanInfo* pTmsInfo = param;
+  pTmsInfo->bNewFileset = true;
+  return;
+}
+
+int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
+  STableMergeScanInfo* pInfo = pOperator->info;
+  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
+  int32_t code = TSDB_CODE_SUCCESS;
+  int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;
+
+  pInfo->bNewFileset = false;
+
+  pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
+  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
+  pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
+                                             pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
+
+  tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
+  tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
+
+  tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL);
+
+  STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
+  param->pOperator = pOperator;
+
+  SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
+  ps->param = param;
+  ps->onlyRef = false;
+  tsortAddSource(pInfo->pSortHandle, ps);
+
+  if (numOfTable == 1) {
+    tsortSetSingleTableMerge(pInfo->pSortHandle);
+  } else {
+    code = tsortOpen(pInfo->pSortHandle);
+  }
+  return code;
+}
+
+void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
+  STableMergeScanInfo* pInfo = pOperator->info;
+
+  SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
+  pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
+  pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer;
+  pInfo->sortExecInfo.loops += sortExecInfo.loops;
+  pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
+  pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
+
+  tsortDestroySortHandle(pInfo->pSortHandle);
+  pInfo->pSortHandle = NULL;
+}
+
 int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
   STableMergeScanInfo* pInfo = pOperator->info;
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
@@ -3348,43 +3423,16 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
 
   tSimpleHashClear(pInfo->mTableNumRows);
 
-  size_t szRow = blockDataGetRowSize(pInfo->pResBlock);   
-//  if (pInfo->mergeLimit != -1) {
-//    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1,
-//                                              NULL, pTaskInfo->id.str, pInfo->mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024);
-//  } else
-  {
-    pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
-    int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
-    pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
-                                              pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0);
-
-    tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
-    tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
-  }
-
-  tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL);
-
-  // one table has one data block
   int32_t numOfTable = tableEndIdx - tableStartIdx + 1;
-
-  STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
-  param->pOperator = pOperator;
   STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx);
   pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock,
-                                (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->mSkipTables);
+                                (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables);
-
+  if (pInfo->filesetDelimited) {
-  SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
+    pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader);
-  ps->param = param;
-  ps->onlyRef = false;
-  tsortAddSource(pInfo->pSortHandle, ps);
-
-  int32_t code = TSDB_CODE_SUCCESS;
-  if (numOfTable == 1) {
-    tsortSetSingleTableMerge(pInfo->pSortHandle);
-  } else {
-    code = tsortOpen(pInfo->pSortHandle);
   }
+  pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo);
+
+  int32_t code = startDurationForGroupTableMergeScan(pOperator);
 
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, terrno);
@@ -3398,21 +3446,13 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) {
   SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
   SStorageAPI*         pAPI = &pTaskInfo->storageAPI;
 
-  SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
+  stopDurationForGroupTableMergeScan(pOperator);
-  pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
-  pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer;
-  pInfo->sortExecInfo.loops += sortExecInfo.loops;
-  pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
-  pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
 
   if (pInfo->base.dataReader != NULL) {
     pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
     pInfo->base.dataReader = NULL;
   }
 
-  tsortDestroySortHandle(pInfo->pSortHandle);
-  pInfo->pSortHandle = NULL;
-
   resetLimitInfoForNextGroup(&pInfo->limitInfo);
   taosHashCleanup(pInfo->mSkipTables);
   pInfo->mSkipTables = NULL;
@@ -3494,6 +3534,10 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
       pBlock->info.id.groupId = pInfo->groupId;
       pOperator->resultInfo.totalRows += pBlock->info.rows;
       return pBlock;
+    } else {
+      if (pInfo->bNewFileset) {
+        stopDurationForGroupTableMergeScan(pOperator);
+        startDurationForGroupTableMergeScan(pOperator);
       } else {
         // Data of this group are all dumped, let's try the next group
         stopGroupTableMergeScan(pOperator);
@@ -3508,6 +3552,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
         resetLimitInfoForNextGroup(&pInfo->limitInfo);
       }
     }
+  }
 
   return pBlock;
 }
@@ -3631,6 +3676,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock);
   pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
 
+  pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
   setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED,
                   pInfo, pTaskInfo);
   pOperator->exprSupp.numOfExprs = numOfCols;

source/libs/executor/src/streameventwindowoperator.c

@@ -297,7 +297,8 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   int32_t rows = pSDataBlock->info.rows;
   blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
   for (int32_t i = 0; i < rows; i += winRows) {
-    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
+    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
+                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i])) {
       i++;
       continue;
     }

source/libs/executor/src/streamtimewindowoperator.c

@@ -450,6 +450,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamInt
   pScanInfo->interval = pInfo->interval;
   pScanInfo->twAggSup = pInfo->twAggSup;
   pScanInfo->pState = pInfo->pState;
+  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;
 }
 
 void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,
@@ -800,7 +801,9 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat
   }
   while (1) {
     bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
-    if ((pInfo->ignoreExpiredData && isClosed && !IS_FINAL_INTERVAL_OP(pOperator)) ||
+    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&
+         checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,
+                          nextWin.ekey)) ||
         !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
       startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
       if (startPos < 0) {
@@ -1623,6 +1626,7 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uin
                                                                 pScanInfo->igCheckUpdate);
   }
   pScanInfo->twAggSup = *pTwSup;
+  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;
 }
 
 static TSKEY sesionTs(void* pKey) {
@@ -2018,7 +2022,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
 
   TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;
   for (int32_t i = 0; i < rows;) {
-    if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
+    if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData &&
+        checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,
+                         pSDataBlock->info.id.uid, endTsCols[i])) {
       i++;
       continue;
     }
@@ -3334,7 +3340,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
   blockDataEnsureCapacity(pAggSup->pScanBlock, rows);
   SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
   for (int32_t i = 0; i < rows; i += winRows) {
-    if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup) || colDataIsNull_s(pKeyColInfo, i)) {
+    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,
+                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i]) || colDataIsNull_s(pKeyColInfo, i)) {
       i++;
       continue;
     }

source/libs/executor/src/timewindowoperator.c

@@ -899,6 +899,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
 
   SWindowRowsSup* pRowSup = &pInfo->winSup;
   pRowSup->numOfRows = 0;
+  pRowSup->startRowIndex = 0;
 
   struct SColumnDataAgg* pAgg = NULL;
   for (int32_t j = 0; j < pBlock->info.rows; ++j) {
@@ -923,9 +924,6 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
       doKeepTuple(pRowSup, tsList[j], gid);
     } else if (compareVal(val, &pInfo->stateKey)) {
       doKeepTuple(pRowSup, tsList[j], gid);
-      if (j == 0 && pRowSup->startRowIndex != 0) {
-        pRowSup->startRowIndex = 0;
-      }
     } else {  // a new state window started
       SResultRow* pResult = NULL;
 

source/libs/function/src/builtins.c

@@ -3425,7 +3425,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_wstart",
     .type = FUNCTION_TYPE_WSTART,
-    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
     .translateFunc = translateTimePseudoColumn,
     .getEnvFunc   = getTimePseudoFuncEnv,
     .initFunc     = NULL,
@@ -3435,7 +3435,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_wend",
     .type = FUNCTION_TYPE_WEND,
-    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
     .translateFunc = translateTimePseudoColumn,
     .getEnvFunc   = getTimePseudoFuncEnv,
     .initFunc     = NULL,
@@ -3445,7 +3445,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
   {
     .name = "_wduration",
     .type = FUNCTION_TYPE_WDURATION,
-    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_SKIP_SCAN_CHECK_FUNC,
     .translateFunc = translateWduration,
     .getEnvFunc   = getTimePseudoFuncEnv,
     .initFunc     = NULL,

source/libs/geometry/src/geomFunc.c

@@ -67,15 +67,19 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) {
     return TSDB_CODE_SUCCESS;
   }
 
-  // make input as a zero ending string
+  char *inputGeom = NULL;
-  char *end = varDataVal(input) + varDataLen(input);
-  char endValue = *end;
-  *end = 0; 
-
   unsigned char *outputGeom = NULL;
   size_t size = 0;
 
-  code = doGeomFromText(varDataVal(input), &outputGeom, &size);
+  // make a zero ending string
+  inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1);
+  if (inputGeom == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  memcpy(inputGeom, varDataVal(input), varDataLen(input));
+
+  code = doGeomFromText(inputGeom, &outputGeom, &size);
   if (code != TSDB_CODE_SUCCESS) {
     goto _exit;
   }
@@ -92,8 +96,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) {
 
 _exit:
   geosFreeBuffer(outputGeom);
-
+  geosFreeBuffer(inputGeom);
-  *end = endValue;  //recover the input string
 
   return code;
 }

source/libs/nodes/src/nodesCloneFuncs.c

@@ -423,6 +423,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
   COPY_SCALAR_FIELD(igLastNull);
   COPY_SCALAR_FIELD(groupOrderScan);
   COPY_SCALAR_FIELD(onlyMetaCtbIdx);
+  COPY_SCALAR_FIELD(filesetDelimited);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -650,6 +651,7 @@ static int32_t physiTableScanCopy(const STableScanPhysiNode* pSrc, STableScanPhy
   COPY_SCALAR_FIELD(triggerType);
   COPY_SCALAR_FIELD(watermark);
   COPY_SCALAR_FIELD(igExpired);
+  COPY_SCALAR_FIELD(filesetDelimited);
   return TSDB_CODE_SUCCESS;
 }
 

source/libs/nodes/src/nodesCodeFuncs.c

@@ -677,6 +677,7 @@ static const char* jkScanLogicPlanDataRequired = "DataRequired";
 static const char* jkScanLogicPlanTagCond = "TagCond";
 static const char* jkScanLogicPlanGroupTags = "GroupTags";
 static const char* jkScanLogicPlanOnlyMetaCtbIdx = "OnlyMetaCtbIdx";
+static const char* jkScanLogicPlanFilesetDelimited = "FilesetDelimited";
 
 static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
@@ -721,6 +722,9 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddBoolToObject(pJson, jkScanLogicPlanOnlyMetaCtbIdx, pNode->onlyMetaCtbIdx);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkScanLogicPlanFilesetDelimited, pNode->filesetDelimited);
+  }
   return code;
 }
 
@@ -768,7 +772,9 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetBoolValue(pJson, jkScanLogicPlanOnlyMetaCtbIdx, &pNode->onlyMetaCtbIdx);
   }
-  
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkScanLogicPlanFilesetDelimited, &pNode->filesetDelimited);
+  }
   return code;
 }
 
@@ -1830,6 +1836,7 @@ static const char* jkTableScanPhysiPlanTags = "Tags";
 static const char* jkTableScanPhysiPlanSubtable = "Subtable";
 static const char* jkTableScanPhysiPlanAssignBlockUid = "AssignBlockUid";
 static const char* jkTableScanPhysiPlanIgnoreUpdate = "IgnoreUpdate";
+static const char* jkTableScanPhysiPlanFilesetDelimited = "FilesetDelimited";
 
 static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj;
@@ -1898,6 +1905,9 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanIgnoreUpdate, pNode->igCheckUpdate);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonAddBoolToObject(pJson, jkTableScanPhysiPlanFilesetDelimited, pNode->filesetDelimited);
+  }
 
   return code;
 }
@@ -1969,6 +1979,9 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) {
   if (TSDB_CODE_SUCCESS == code) {
     code = tjsonGetTinyIntValue(pJson, jkTableScanPhysiPlanIgnoreUpdate, &pNode->igCheckUpdate);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tjsonGetBoolValue(pJson, jkTableScanPhysiPlanFilesetDelimited, &pNode->filesetDelimited);
+  }
 
   return code;
 }

source/libs/nodes/src/nodesMsgFuncs.c

@@ -2167,7 +2167,9 @@ static int32_t physiTableScanNodeInlineToMsg(const void* pObj, STlvEncoder* pEnc
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvEncodeValueI8(pEncoder, pNode->igCheckUpdate);
   }
-
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvEncodeValueBool(pEncoder, pNode->filesetDelimited);
+  }
   return code;
 }
 
@@ -2246,6 +2248,9 @@ static int32_t msgToPhysiTableScanNodeInline(STlvDecoder* pDecoder, void* pObj)
   if (TSDB_CODE_SUCCESS == code) {
     code = tlvDecodeValueI8(pDecoder, &pNode->igCheckUpdate);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    code = tlvDecodeValueBool(pDecoder, &pNode->filesetDelimited);
+  }
 
   return code;
 }

source/libs/parser/src/parInsertSql.c

@@ -268,26 +268,47 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E
   return code;
 }
 
-static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) {
+static int parseTimestampOrInterval(const char** end, SToken* pToken, int16_t timePrec, int64_t* ts, int64_t* interval, SMsgBuf* pMsgBuf, bool* isTs) {
-  int32_t     index = 0;
-  int64_t     interval;
-  int64_t     ts = 0;
-  const char* pTokenEnd = *end;
-
   if (pToken->type == TK_NOW) {
-    ts = taosGetTimestamp(timePrec);
+    *isTs = true;
+    *ts = taosGetTimestamp(timePrec);
   } else if (pToken->type == TK_TODAY) {
-    ts = taosGetTimestampToday(timePrec);
+    *isTs = true;
+    *ts = taosGetTimestampToday(timePrec);
   } else if (pToken->type == TK_NK_INTEGER) {
-    if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &ts)) {
+    *isTs = true;
+    if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, ts)) {
       return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
     }
+  } else if (pToken->type == TK_NK_VARIABLE) {
+    char unit = 0;
+    *isTs = false;
+    if (parseAbsoluteDuration(pToken->z, pToken->n, interval, &unit, timePrec) != TSDB_CODE_SUCCESS) {
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
   } else {  // parse the RFC-3339/ISO-8601 timestamp format string
-    if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
+    *isTs = true;
+    if (taosParseTime(pToken->z, ts, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
       return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
     }
+  }
 
   return TSDB_CODE_SUCCESS;
+}
+
+static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) {
+  int32_t     index = 0, i = 0;
+  int64_t     interval = 0, tempInterval = 0;
+  int64_t     ts = 0, tempTs = 0;
+  bool firstIsTS = false, secondIsTs = false;
+  const char* pTokenEnd = *end;
+
+  if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) {
+    return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+  }
+
+  if (firstIsTS) {
+    *time = ts;
   }
 
   for (int k = pToken->n; pToken->z[k] != '\0'; k++) {
@@ -299,45 +320,98 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t
     }
     if (pToken->z[k] == ',') {
       *end = pTokenEnd;
-      *time = ts;
+      if (!firstIsTS) {
-      return 0;
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+      }
+      *time = ts;
+      return TSDB_CODE_SUCCESS;
     }
-
     break;
   }
 
-  /*
+  while (pTokenEnd[i] != '\0') {
-   * time expression:
+    if (pTokenEnd[i] == ' ' || pTokenEnd[i] == '\t') {
-   * e.g., now+12a, now-5h
+      i++;
-   */
+      continue;
+    }
+    else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') {
+      *end = pTokenEnd + i;
+      if (!firstIsTS) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+      }
+      *time = ts;
+      return TSDB_CODE_SUCCESS;
+    } else {
+      break;
+    }
+  }
+  pTokenEnd = pTokenEnd + i;
+
   index = 0;
   SToken token = tStrGetToken(pTokenEnd, &index, false, NULL);
-  pTokenEnd += index;
 
   if (token.type == TK_NK_MINUS || token.type == TK_NK_PLUS) {
+    pTokenEnd += index;
     index = 0;
     SToken valueToken = tStrGetToken(pTokenEnd, &index, false, NULL);
     pTokenEnd += index;
+    char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
+    if (TK_NK_STRING == valueToken.type) {
+      if (valueToken.n >= TSDB_MAX_BYTES_PER_ROW) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", valueToken.z);
+      }
+      int32_t len = trimString(valueToken.z, valueToken.n, tmpTokenBuf, TSDB_MAX_BYTES_PER_ROW);
+      valueToken.z = tmpTokenBuf;
+      valueToken.n = len;
+    }
+
+    if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) {
+      return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+    }
 
     if (valueToken.n < 2) {
       return buildSyntaxErrMsg(pMsgBuf, "value expected in timestamp", token.z);
     }
 
-    char unit = 0;
+    if (secondIsTs) {
-    if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval, &unit, timePrec) != TSDB_CODE_SUCCESS) {
+      // not support operator between tow timestamp, such as today() + now()
-      return TSDB_CODE_TSC_INVALID_OPERATION;
+      if (firstIsTS) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+      }
+      ts = tempTs;
+    }else {
+      // not support operator between tow interval, such as 2h + 3s
+      if (!firstIsTS) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+      }
+      interval = tempInterval;
+    }
+    if (token.type == TK_NK_MINUS) {
+      // not support interval - ts,such as 2h - today()
+      if (secondIsTs) {
+        return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+      }
+      *time = ts - interval;
+    } else {
+      *time = ts + interval;
     }
 
-    if (token.type == TK_NK_PLUS) {
+    for (int k = valueToken.n; valueToken.z[k] != '\0'; k++) {
-      ts += interval;
+      if (valueToken.z[k] == ' ' || valueToken.z[k] == '\t') continue;
-    } else {
+      if (valueToken.z[k] == '(' && valueToken.z[k + 1] == ')') {  // for insert NOW()/TODAY()
-      ts = ts - interval;
+        *end = pTokenEnd = &valueToken.z[k + 2];
+        k++;
+        continue;
+      }
+      if (valueToken.z[k] == ',' || valueToken.z[k] == ')') {
+        *end = pTokenEnd;
+        return TSDB_CODE_SUCCESS;
+      }
+      return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z);
+    }
   }
 
   *end = pTokenEnd;
-  }
-
-  *time = ts;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -433,7 +507,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_TINYINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
       } else if (!IS_VALID_TINYINT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
@@ -444,7 +519,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
       } else if (uv > UINT8_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
@@ -454,7 +530,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
       } else if (!IS_VALID_SMALLINT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
@@ -464,7 +541,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
       } else if (uv > UINT16_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
@@ -474,7 +552,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_INT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
       } else if (!IS_VALID_INT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
@@ -484,7 +563,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
       } else if (uv > UINT32_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
@@ -494,7 +574,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_BIGINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
       }
       val->i64 = iv;
@@ -502,7 +583,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
       }
       *(uint64_t*)(&val->i64) = uv;
@@ -511,11 +593,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
 
     case TSDB_DATA_TYPE_FLOAT: {
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
+      if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
-          isnan(dv)) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
       *(float*)(&val->i64) = dv;
@@ -524,8 +606,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
 
     case TSDB_DATA_TYPE_DOUBLE: {
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
-        return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
+      if (TSDB_CODE_SUCCESS != code) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
       if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
@@ -626,7 +709,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm
 
 static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken,
                              SArray* pTagName, SArray* pTagVals, STag** pTag) {
-  if (!isNullValue(pTagSchema->type, pToken)) {
+  bool isNull = isNullValue(pTagSchema->type, pToken);
+  if (!isNull) {
     taosArrayPush(pTagName, pTagSchema->name);
   }
 
@@ -635,13 +719,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm
       return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z);
     }
 
-    if (isNullValue(pTagSchema->type, pToken)) {
+    if (isNull) {
       return tTagNew(pTagVals, 1, true, pTag);
     } else {
       return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg);
     }
   }
 
+  if (isNull) return 0;
+
   STagVal val = {0};
   int32_t code =
       parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg);
@@ -671,7 +757,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs
   if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER &&
        pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL &&
        pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT &&
-       pToken->type != TK_NK_BIN) ||
+       pToken->type != TK_NK_BIN && pToken->type != TK_NK_VARIABLE) ||
       (pToken->n == 0) || (pToken->type == TK_NK_RP)) {
     return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z);
   }
@@ -850,6 +936,9 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt
     SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]];
     isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON;
     code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg);
+    if (TK_NK_VARIABLE == token.type) {
+      code = buildSyntaxErrMsg(&pCxt->msg, "not expected tags values ", token.z);
+    }
     if (TSDB_CODE_SUCCESS == code) {
       code = parseTagValue(pCxt, pStmt, &pStmt->pSql, pTagSchema, &token, pTagName, pTagVals, &pTag);
     }
@@ -1351,7 +1440,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_TINYINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z);
       } else if (!IS_VALID_TINYINT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z);
@@ -1359,7 +1449,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z);
       } else if (pVal->value.val > UINT8_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z);
@@ -1367,7 +1458,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z);
       } else if (!IS_VALID_SMALLINT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z);
@@ -1375,7 +1467,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z);
       } else if (pVal->value.val > UINT16_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z);
@@ -1383,7 +1476,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_INT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z);
       } else if (!IS_VALID_INT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z);
@@ -1391,7 +1485,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_UINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z);
       } else if (pVal->value.val > UINT32_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z);
@@ -1399,25 +1494,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_BIGINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z);
       }
       break;
     }
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z);
       }
       break;
     }
     case TSDB_DATA_TYPE_FLOAT: {
-      char*  endptr = NULL;
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
+      if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
-          isnan(dv)) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
       float f = dv;
@@ -1425,12 +1521,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
-      char*  endptr = NULL;
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
-        return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
+      if (TSDB_CODE_SUCCESS != code) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
+      if (isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
       }
       pVal->value.val = *(int64_t*)&dv;
@@ -1540,6 +1636,10 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo
     if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
       return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z);
     }
+
+    if (TK_NK_VARIABLE == pToken->type && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) {
+      return buildSyntaxErrMsg(&pCxt->msg, "invalid values", pToken->z);
+    }
     pVal->flag = CV_FLAG_NULL;
     return TSDB_CODE_SUCCESS;
   }
@@ -1592,6 +1692,9 @@ typedef union SRowsDataContext{
 static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) {
   *pFoundCtbName = false;
   int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
+  if (TK_NK_VARIABLE == pToken->type) {
+    code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname");
+  }
   if (code == TSDB_CODE_SUCCESS){
     if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) {
       return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value");
@@ -1629,6 +1732,10 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif
       SToken*  pTagToken = (SToken*)(tagTokens + i);
       SSchema* pTagSchema = tagSchemas[i];
       code = checkAndTrimValue(pTagToken, pCxt->tmpTokenBuf, &pCxt->msg);
+      if (TK_NK_VARIABLE == pTagToken->type) {
+        code = buildInvalidOperationMsg(&pCxt->msg, "not expected tag");
+      }
+
       if (code == TSDB_CODE_SUCCESS) {
         code = parseTagValue(pCxt, pStmt, NULL, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals,
                              &pStbRowsCxt->pTag);
@@ -1673,6 +1780,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
       const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]];
       SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]);
       code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal);
+      if (TK_NK_VARIABLE == pToken->type) {
+          code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value");
+      }
     } else if (pCols->pColIndex[i] < tbnameIdx) {
       const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]];
       if (canParseTagsAfter) {
@@ -1681,6 +1791,9 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
         ++(*pNumOfTagTokens);
       } else {
         code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg);
+        if (TK_NK_VARIABLE == pToken->type) {
+          code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value");
+        }
         if (code == TSDB_CODE_SUCCESS) {
           code = parseTagValue(pCxt, pStmt, ppSql, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag);
         }