Merge branch '3.0' of github.com:taosdata/TDengine into szhou/tms-duration

.gitignore

@@ -11,6 +11,7 @@ CMakeSettings.json
 cmake-build-debug/
 cmake-build-release/
 cscope.out
+cscope.files
 .DS_Store
 debug/
 release/

cmake/cmake.define

@@ -181,17 +181,17 @@ ELSE ()
         ENDIF()
         MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
 
-        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
-            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
-        ENDIF()
-
-        IF (COMPILER_SUPPORT_AVX512VL)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
-        MESSAGE(STATUS "avx512vl supported by compiler")
-        ENDIF()
+#        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
+#            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
+#        ENDIF()
+#
+#        IF (COMPILER_SUPPORT_AVX512VL)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
+#        MESSAGE(STATUS "avx512vl supported by compiler")
+#        ENDIF()
     ENDIF()
 
     # build mode

docs/examples/python/bind_param_example.py

@@ -20,7 +20,7 @@ def get_ts(ts: str):
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/conn_websocket_pandas.py

@@ -4,7 +4,7 @@ import taos
 
 taos_conn = taos.connect()
 taos_conn.execute('drop database if exists power')
-taos_conn.execute('create database if not exists power wal_retention_period 3600')
+taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ')
 taos_conn.execute("use power")
 taos_conn.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")

docs/examples/python/connect_rest_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power")
-cursor.execute("CREATE DATABASE power")
+cursor.execute("CREATE DATABASE power keep 36500 ")
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")
 

docs/examples/python/connect_rest_with_req_id_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power", req_id=1)
-cursor.execute("CREATE DATABASE power", req_id=2)
+cursor.execute("CREATE DATABASE power keep 36500", req_id=2)
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3)
 

docs/examples/python/connect_websocket_examples.py

@@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy")
-conn.execute("create database if not exists connwspy wal_retention_period 3600")
+conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ")
 conn.execute("use connwspy")
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)")
 conn.execute("create table if not exists tb1 using stb tags (1)")
 conn.execute("insert into tb1 values (now, 1)")
-conn.execute("insert into tb1 values (now, 2)")
-conn.execute("insert into tb1 values (now, 3)")
+conn.execute("insert into tb1 values (now+1s, 2)")
+conn.execute("insert into tb1 values (now+2s, 3)")
 
 r = conn.execute("select * from stb")
 result = conn.query("select * from stb")

docs/examples/python/connect_websocket_with_req_id_examples.py

@@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy", req_id=1)
-conn.execute("create database if not exists connwspy", req_id=2)
+conn.execute("create database if not exists connwspy keep 36500", req_id=2)
 conn.execute("use connwspy", req_id=3)
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4)
 conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5)

docs/examples/python/connection_usage_native_reference.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")

docs/examples/python/connection_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)

docs/examples/python/cursor_usage_native_reference.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test")
-cursor.execute("CREATE DATABASE test")
+cursor.execute("CREATE DATABASE test keep 36500")
 cursor.execute("USE test")
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 

docs/examples/python/cursor_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test", req_id=1)
-cursor.execute("CREATE DATABASE test", req_id=2)
+cursor.execute("CREATE DATABASE test keep 36500", req_id=2)
 cursor.execute("USE test", req_id=3)
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4)
 

docs/examples/python/fast_write_example.py

@@ -160,7 +160,7 @@ def main(infinity):
 
     conn = get_connection()
     conn.execute("DROP DATABASE IF EXISTS test")
-    conn.execute("CREATE DATABASE IF NOT EXISTS test")
+    conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500")
     conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")
     conn.close()

docs/examples/python/json_protocol_example.py

@@ -16,7 +16,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/kafka_example_common.py

@@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD
              'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
              'California.SantaClara', 'California.Cupertino']
 
-CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
+CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
 USE_DATABASE_SQL = 'use {}'
 DROP_TABLE_SQL = 'drop table if exists meters'
 DROP_DATABASE_SQL = 'drop database if exists {}'

docs/examples/python/line_protocol_example.py

@@ -15,7 +15,7 @@ def get_connection():
 
 def create_database(conn):
     # the default precision is ms (microsecond), but we use us(microsecond) here.
-    conn.execute("CREATE DATABASE test precision 'us'")
+    conn.execute("CREATE DATABASE test precision 'us' keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/multi_bind_example.py

@@ -71,7 +71,7 @@ def insert_data():
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/native_insert_example.py

@@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection:
 
 
 def create_stable(conn: taos.TaosConnection):
-    conn.execute("CREATE DATABASE power")
+    conn.execute("CREATE DATABASE power keep 36500")
     conn.execute("USE power")
     conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")

docs/examples/python/result_set_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 # prepare data

docs/examples/python/result_set_with_req_id_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)
 # prepare data

docs/examples/python/schemaless_insert.py

@@ -3,7 +3,7 @@ import taos
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_raw.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_req_id.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_ttl.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_req_id.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_ttl.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/sql_writer.py

@@ -10,7 +10,7 @@ class SQLWriter:
         self._tb_tags = {}
         self._conn = get_connection_func()
         self._max_sql_length = self.get_max_sql_length()
-        self._conn.execute("create database if not exists test")
+        self._conn.execute("create database if not exists test keep 36500")
         self._conn.execute("USE test")
 
     def get_max_sql_length(self):

docs/examples/python/stmt_example.py

@@ -10,7 +10,7 @@ db_name = 'test_ws_stmt'
 def before():
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/stmt_websocket_example.py

@@ -9,7 +9,7 @@ import taos
 def before_test(db_name):
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/telnet_line_protocol_example.py

@@ -19,7 +19,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/tmq_assignment_example.py

@@ -7,7 +7,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

docs/examples/python/tmq_example.py

@@ -6,7 +6,7 @@ def init_tmq_env(db, topic):
     conn = taos.connect()
     conn.execute("drop topic if exists {}".format(topic))
     conn.execute("drop database if exists {}".format(db))
-    conn.execute("create database if not exists {} wal_retention_period 3600".format(db))
+    conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db))
     conn.select_db(db)
     conn.execute(
         "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))")

docs/examples/python/tmq_websocket_assgnment_example.py

@@ -6,7 +6,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

include/common/tmsg.h

@@ -295,7 +295,36 @@ typedef enum ENodeType {
   QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_GRANT_STMT,
   QUERY_NODE_REVOKE_STMT,
-  QUERY_NODE_SHOW_DNODES_STMT,
+  // placeholder for [152, 180]
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
+  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_DELETE_STMT,
+  QUERY_NODE_INSERT_STMT,
+  QUERY_NODE_QUERY,
+  QUERY_NODE_SHOW_DB_ALIVE_STMT,
+  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_RESTORE_DNODE_STMT,
+  QUERY_NODE_RESTORE_QNODE_STMT,
+  QUERY_NODE_RESTORE_MNODE_STMT,
+  QUERY_NODE_RESTORE_VNODE_STMT,
+  QUERY_NODE_PAUSE_STREAM_STMT,
+  QUERY_NODE_RESUME_STREAM_STMT,
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
+
+  // show statement nodes
+  // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
+  QUERY_NODE_SHOW_DNODES_STMT = 400,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
@@ -324,31 +353,6 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_VIEWS_STMT,
-  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
-  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
-  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
-  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
-  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
-  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
-  QUERY_NODE_SHOW_SCORES_STMT,
-  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
-  QUERY_NODE_KILL_CONNECTION_STMT,
-  QUERY_NODE_KILL_QUERY_STMT,
-  QUERY_NODE_KILL_TRANSACTION_STMT,
-  QUERY_NODE_DELETE_STMT,
-  QUERY_NODE_INSERT_STMT,
-  QUERY_NODE_QUERY,
-  QUERY_NODE_SHOW_DB_ALIVE_STMT,
-  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
-  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
-  QUERY_NODE_RESTORE_DNODE_STMT,
-  QUERY_NODE_RESTORE_QNODE_STMT,
-  QUERY_NODE_RESTORE_MNODE_STMT,
-  QUERY_NODE_RESTORE_VNODE_STMT,
-  QUERY_NODE_PAUSE_STREAM_STMT,
-  QUERY_NODE_RESUME_STREAM_STMT,
-  QUERY_NODE_CREATE_VIEW_STMT,
-  QUERY_NODE_DROP_VIEW_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,

include/common/tmsgdef.h

@@ -26,10 +26,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) "null",
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp",
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   char *tMsgInfo[] = {
 
@@ -37,20 +37,20 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) 
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
   int32_t tMsgRangeDict[] = {
 
 #elif defined(TD_MSG_NUMBER_)
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -58,10 +58,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(type)
+  #define TD_CLOSE_MSG_SEG(type)
 
   int32_t tMsgDict[] = {
 
@@ -70,10 +70,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -82,10 +82,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
 
   enum { // WARN: new msg should be appended to segment tail
 #endif
@@ -109,7 +109,7 @@
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
@@ -218,7 +218,7 @@
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@@ -268,7 +268,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
@@ -283,7 +283,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
@@ -301,11 +301,11 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
@@ -337,7 +337,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
@@ -348,7 +348,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
@@ -362,10 +362,10 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
-  
+  TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG) 
 
   
 

include/common/ttypes.h

@@ -275,9 +275,11 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT64(_t)     ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
+#define IS_VALID_UINT64(_t)    ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -37,6 +37,10 @@ typedef struct SVariant {
   };
 } SVariant;
 
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value);
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value);
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 

include/libs/sync/sync.h

@@ -289,6 +289,9 @@ const char* syncStr(ESyncState state);
 
 int32_t    syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
 
+// util
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -288,6 +288,7 @@ typedef enum ELogicConditionType {
 #define TSDB_CONN_ACTIVE_KEY_LEN 255
 
 #define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024)
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120  // default payload size, greater than PATH_MAX value

source/client/inc/clientSml.h

@@ -65,14 +65,32 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-//#define TS        "_ts"
-//#define TS_LEN    3
 #define VALUE     "_value"
-#define VALUE_LEN 6
+#define VALUE_LEN (sizeof(VALUE)-1)
 
 #define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 10
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
+
+#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+
+#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
+
+#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
+
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
 
 typedef enum {
   SCHEMA_ACTION_NULL,
@@ -83,18 +101,6 @@ typedef enum {
   SCHEMA_ACTION_CHANGE_TAG_SIZE,
 } ESchemaAction;
 
-typedef struct {
-  const void  *key;
-  int32_t      keyLen;
-  void        *value;
-  bool         used;
-}Node;
-
-typedef struct NodeList{
-  Node             data;
-  struct NodeList* next;
-}NodeList;
-
 typedef struct {
   char *measure;
   char *tags;
@@ -117,7 +123,6 @@ typedef struct {
   int32_t     sTableNameLen;
   char        childTableName[TSDB_TABLE_NAME_LEN];
   uint64_t    uid;
-//  void       *key;        // for openTsdb
 
   SArray *tags;
 
@@ -161,7 +166,8 @@ typedef struct {
 typedef struct {
   int64_t id;
 
-  SMLProtocolType protocol;
+  TSDB_SML_PROTOCOL_TYPE protocol;
+
   int8_t          precision;
   bool            reRun;
   bool            dataFormat;  // true means that the name and order of keys in each line are the same(only for influx protocol)
@@ -201,29 +207,8 @@ typedef struct {
   bool         needModifySchema;
 } SSmlHandle;
 
-#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
-
-#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
-
-#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
-
-#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
-
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
-
-#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
-  for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
-      MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      keyLen--;                              \
-    }                                        \
-  }
-
-extern int64_t smlFactorNS[3];
-extern int64_t smlFactorS[3];
+extern int64_t smlFactorNS[];
+extern int64_t smlFactorS[];
 
 typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
@@ -231,16 +216,10 @@ SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
 int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-//int           nodeListSize(NodeList* list);
-bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
 int64_t       smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
-int8_t        smlGetTsTypeByLen(int32_t len);
 SSmlTableInfo*    smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
 SSmlSTableMeta*   smlBuildSTableMeta(bool isDataFormat);
 int32_t           smlSetCTableName(SSmlTableInfo *oneTable);
@@ -253,12 +232,45 @@ int32_t           smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
 uint8_t           smlGetTimestampLen(int64_t num);
 void              smlDestroyTableInfo(void *para);
 
-void freeSSmlKv(void* data);
+void    freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
-void    smlStrReplace(char* src, int32_t len);
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
+bool            isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+bool            isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+int32_t         smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlJoinMeasureTag(SSmlLineInfo *elements);
+void            smlBuildTsKv(SSmlKv *kv, int64_t ts);
+int32_t         smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
+int32_t         smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
+
+static inline bool smlDoubleToInt64OverFlow(double num) {
+  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
+  return false;
+}
+
+static inline void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
+static inline int8_t smlGetTsTypeByLen(int32_t len) {
+  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
+    return TSDB_TIME_PRECISION_SECONDS;
+  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+    return TSDB_TIME_PRECISION_MILLI;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -732,7 +732,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         return -1;
       }
       newstr[0] = '"';
-      strncpy(newstr+1, str, len);
+      memcpy(newstr+1, str, len);
       newstr[len + 1] = '"';
       newstr[len + 2] = '\0';
       str = newstr;

source/client/src/clientSml.c

@@ -20,75 +20,93 @@
 
 #include "clientSml.h"
 
-int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
-int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
-int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
+#define RETURN_FALSE                                 \
+  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
+  return false;
 
-//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
-//  NodeList *tmp = list;
-//  while (tmp) {
-//    if (fn == NULL) {
-//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return tmp->data.value;
-//      }
-//    } else {
-//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
-//        return tmp->data.value;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  return NULL;
-//}
-//
-//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
-//  NodeList *tmp = *list;
-//  while (tmp) {
-//    if (!tmp->data.used) break;
-//    if (fn == NULL) {
-//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return -1;
-//      }
-//    } else {
-//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
-//        return -1;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  if (tmp) {
-//    tmp->data.key = key;
-//    tmp->data.keyLen = len;
-//    tmp->data.value = value;
-//    tmp->data.used = true;
-//  } else {
-//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
-//    if (newNode == NULL) {
-//      return -1;
-//    }
-//    newNode->data.key = key;
-//    newNode->data.keyLen = len;
-//    newNode->data.value = value;
-//    newNode->data.used = true;
-//    newNode->next = *list;
-//    *list = newNode;
-//  }
-//  return 0;
-//}
-//
-//int nodeListSize(NodeList *list) {
-//  int cnt = 0;
-//  while (list) {
-//    if (list->data.used)
-//      cnt++;
-//    else
-//      break;
-//    list = list->next;
-//  }
-//  return cnt;
-//}
+#define SET_DOUBLE                     \
+  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
+  kvVal->d = result;
+
+#define SET_FLOAT                                                                              \
+  if (!IS_VALID_FLOAT(result)) {                                                               \
+    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
+    return false;                                                                              \
+  }                                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
+  kvVal->f = (float)result;
+
+#define SET_BIGINT                                                                                       \
+  errno = 0;                                                                                             \
+  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
+  if (errno == ERANGE) {                                                                                 \
+    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
+    return false;                                                                                        \
+  }                                                                                                      \
+  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
+  kvVal->i = tmp;
+
+#define SET_INT                                                                    \
+  if (!IS_VALID_INT(result)) {                                                     \
+    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
+    return false;                                                                  \
+  }                                                                                \
+  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
+  kvVal->i = result;
+
+#define SET_SMALL_INT                                                          \
+  if (!IS_VALID_SMALLINT(result)) {                                            \
+    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
+    return false;                                                              \
+  }                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
+  kvVal->i = result;
+
+#define SET_UBIGINT                                                                             \
+  errno = 0;                                                                                    \
+  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
+  if (errno == ERANGE || result < 0) {                                                          \
+    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
+    return false;                                                                               \
+  }                                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
+  kvVal->u = tmp;
+
+#define SET_UINT                                                                  \
+  if (!IS_VALID_UINT(result)) {                                                   \
+    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
+  kvVal->u = result;
+
+#define SET_USMALL_INT                                                            \
+  if (!IS_VALID_USMALLINT(result)) {                                              \
+    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
+  kvVal->u = result;
+
+#define SET_TINYINT                                                       \
+  if (!IS_VALID_TINYINT(result)) {                                        \
+    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
+    return false;                                                         \
+  }                                                                       \
+  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
+  kvVal->i = result;
+
+#define SET_UTINYINT                                                            \
+  if (!IS_VALID_UTINYINT(result)) {                                             \
+    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
+    return false;                                                               \
+  }                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
+  kvVal->u = result;
+
+int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
+int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
 
 static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
   SUserAuthInfo pAuth = {0};
@@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
-inline bool smlDoubleToInt64OverFlow(double num) {
-  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
-  return false;
-}
-
-void smlStrReplace(char* src, int32_t len){
-  if (!tsSmlDot2Underline) return;
-  for(int i = 0; i < len; i++){
-    if(src[i] == '.'){
-      src[i] = '_';
-    }
-  }
-}
 
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
@@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
   return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
 }
 
-int8_t smlGetTsTypeByLen(int32_t len) {
-  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
-    return TSDB_TIME_PRECISION_SECONDS;
-  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-    return TSDB_TIME_PRECISION_MILLI;
-  } else {
-    return -1;
-  }
-}
-
 SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
   SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
   if (!tag) {
@@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu
     goto cleanup;
   }
 
-  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
-  //  if (tag->tags == NULL) {
-  //    uError("SML:smlBuildTableInfo failed to allocate memory");
-  //    goto cleanup;
-  //  }
   return tag;
 
 cleanup:
@@ -192,6 +182,242 @@ cleanup:
   return NULL;
 }
 
+void smlBuildTsKv(SSmlKv *kv, int64_t ts){
+  kv->key = tsSmlTsDefaultName;
+  kv->keyLen = strlen(tsSmlTsDefaultName);
+  kv->type = TSDB_DATA_TYPE_TIMESTAMP;
+  kv->i = ts;
+  kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){
+  SSmlSTableMeta* sMeta = NULL;
+  char *measure = currElement->measure;
+  int   measureLen = currElement->measureLen;
+  if (currElement->measureEscaped) {
+    measure = (char *)taosMemoryMalloc(measureLen);
+    memcpy(measure, currElement->measure, measureLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+  }
+  STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+  if (currElement->measureEscaped) {
+    taosMemoryFree(measure);
+  }
+  if (pTableMeta == NULL) {
+    info->dataFormat = false;
+    info->reRun = true;
+    terrno = TSDB_CODE_SUCCESS;
+    return sMeta;
+  }
+  sMeta = smlBuildSTableMeta(info->dataFormat);
+  if(sMeta == NULL){
+    taosMemoryFreeClear(pTableMeta);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return sMeta;
+  }
+  sMeta->tableMeta = pTableMeta;
+  taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+  for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
+    SSchema *col = pTableMeta->schema + i;
+    SSmlKv   kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
+    if (col->type == TSDB_DATA_TYPE_NCHAR) {
+      kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) {
+      kv.length = col->bytes - VARSTR_HEADER_SIZE;
+    } else{
+      kv.length = col->bytes;
+    }
+
+    if(i < pTableMeta->tableInfo.numOfColumns){
+      taosArrayPush(sMeta->cols, &kv);
+    }else{
+      taosArrayPush(sMeta->tags, &kv);
+    }
+  }
+  return sMeta;
+}
+
+bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  // cnt begin 0, add ts so + 2
+  if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  // bind data
+  int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
+  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+    uDebug("smlBuildCol error, retry");
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  if (cnt >= taosArrayGetSize(info->maxColKVs)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+
+  if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+}
+
+bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+    goto END;
+  }
+
+  if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    goto END;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    goto END;
+  }
+
+  if (unlikely(kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+
+END:
+  info->dataFormat = false;
+  info->reRun = true;
+  return false;
+}
+
+int32_t smlJoinMeasureTag(SSmlLineInfo *elements){
+  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+  if(elements->measureTag == NULL){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
+  bool isSameMeasure = IS_SAME_SUPER_TABLE;
+  if(isSameMeasure) {
+    return 0;
+  }
+  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+
+  SSmlSTableMeta *sMeta = NULL;
+  if (unlikely(tmp == NULL)) {
+    sMeta = smlBuildSuperTableInfo(info, elements);
+    if(sMeta == NULL) return -1;
+  }else{
+    sMeta = *tmp;
+  }
+  ASSERT(sMeta != NULL);
+  info->currSTableMeta = sMeta->tableMeta;
+  info->maxTagKVs = sMeta->tags;
+  info->maxColKVs = sMeta->cols;
+  return 0;
+}
+
+int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
+  SSmlTableInfo **oneTable =
+      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
+  SSmlTableInfo *tinfo = NULL;
+  if (unlikely(oneTable == NULL)) {
+    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
+    if (unlikely(!tinfo)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES);
+
+    tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
+    for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
+      SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
+      if (kv->keyEscaped) kv->key = NULL;
+      if (kv->valueEscaped) kv->value = NULL;
+    }
+
+    smlSetCTableName(tinfo);
+    getTableUid(info, elements, tinfo);
+    if (info->dataFormat) {
+      info->currSTableMeta->uid = tinfo->uid;
+      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
+      if (tinfo->tableDataCtx == NULL) {
+        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
+        smlDestroyTableInfo(&tinfo);
+        return TSDB_CODE_SML_INVALID_DATA;
+      }
+    }
+  }else{
+    tinfo = *oneTable;
+  }
+  ASSERT(tinfo != NULL);
+  if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1);
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i);
+    if (elements->colArray == NULL) {
+      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
+    }
+    taosArrayPush(elements->colArray, kvTs);
+    taosArrayPush(elements->colArray, kv);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i);
+    taosArraySet(elements->colArray, 0, kvTs);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
   bool autoChildName = false;
   size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
@@ -328,98 +554,6 @@ cleanup:
   return NULL;
 }
 
-// uint16_t smlCalTypeSum(char* endptr, int32_t left){
-//   uint16_t sum = 0;
-//   for(int i = 0; i < left; i++){
-//     sum += endptr[i];
-//   }
-//   return sum;
-// }
-
-#define RETURN_FALSE                                 \
-  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-  return false;
-
-#define SET_DOUBLE                     \
-  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-  kvVal->d = result;
-
-#define SET_FLOAT                                                                              \
-  if (!IS_VALID_FLOAT(result)) {                                                               \
-    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-    return false;                                                                              \
-  }                                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-  kvVal->f = (float)result;
-
-#define SET_BIGINT                                                                                       \
-  errno = 0;                                                                                             \
-  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-  if (errno == ERANGE) {                                                                                 \
-    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-    return false;                                                                                        \
-  }                                                                                                      \
-  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-  kvVal->i = tmp;
-
-#define SET_INT                                                                    \
-  if (!IS_VALID_INT(result)) {                                                     \
-    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-    return false;                                                                  \
-  }                                                                                \
-  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-  kvVal->i = result;
-
-#define SET_SMALL_INT                                                          \
-  if (!IS_VALID_SMALLINT(result)) {                                            \
-    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-    return false;                                                              \
-  }                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-  kvVal->i = result;
-
-#define SET_UBIGINT                                                                             \
-  errno = 0;                                                                                    \
-  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-  if (errno == ERANGE || result < 0) {                                                          \
-    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-    return false;                                                                               \
-  }                                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-  kvVal->u = tmp;
-
-#define SET_UINT                                                                  \
-  if (!IS_VALID_UINT(result)) {                                                   \
-    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-  kvVal->u = result;
-
-#define SET_USMALL_INT                                                            \
-  if (!IS_VALID_USMALLINT(result)) {                                              \
-    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-  kvVal->u = result;
-
-#define SET_TINYINT                                                       \
-  if (!IS_VALID_TINYINT(result)) {                                        \
-    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
-    return false;                                                         \
-  }                                                                       \
-  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
-  kvVal->i = result;
-
-#define SET_UTINYINT                                                            \
-  if (!IS_VALID_UTINYINT(result)) {                                             \
-    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
-    return false;                                                               \
-  }                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
-  kvVal->u = result;
-
 bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
   const char *pVal = kvVal->value;
   int32_t     len = kvVal->length;
@@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
 static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                               ESchemaAction action) {
   SRequestObj   *pRequest = NULL;
@@ -1121,35 +1253,6 @@ end:
   return code;
 }
 
-/*
-static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
-  for(int i = 0; i < taosArrayGetSize(tags); i++) {
-    SSmlKv *tag = taosArrayGet(tags, i);
-    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
-      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
-      return TSDB_CODE_TSC_DUP_NAMES;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
-  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-
-  end:
-  taosHashCleanup(dumplicateKey);
-  return ret;
-}
-*/
-
 static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) {
     taosHashCleanup(kvHash);
   }
 
-  //  if (info->parseJsonByLib) {
-  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
-  //    if (key != NULL) taosMemoryFree(key->tags);
-  //  }
-  //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
@@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
 
-  // destroy info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // destroy info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
-
-  // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
   taosHashCleanup(info->childTables);
   taosHashCleanup(info->superTables);
@@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) {
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
-      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+      if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) {
+        taosMemoryFree(info->lines[i].measureTag);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
         return ret;
       }
     } else {
-      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
-      //      if (ret != TSDB_CODE_SUCCESS) {
-      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
-      //        return ret;
-      //      }
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
@@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
   uDebug(
       "SML:0x%" PRIx64
       " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
-        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
-      "",
+        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64,
       info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
       info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
       info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
       info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
       info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
+
 }
 
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
-  // clear info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(info, *oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // clear info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);

source/client/src/clientSmlJson.c

@@ -29,199 +29,6 @@
       (start)++;                 \
   }
 
-// SArray *smlJsonParseTags(char *start, char *end){
-//   SArray *tags = taosArrayInit(4, sizeof(SSmlKv));
-//   while(start < end){
-//     SSmlKv kv = {0};
-//     kv.type = TSDB_DATA_TYPE_NCHAR;
-//     bool isInQuote = false;
-//     while(start < end){
-//       if(unlikely(!isInQuote && *start == '"')){
-//         start++;
-//         kv.key = start;
-//         isInQuote = true;
-//         continue;
-//       }
-//       if(unlikely(isInQuote && *start == '"')){
-//         kv.keyLen = start - kv.key;
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//     bool hasColon = false;
-//     while(start < end){
-//       if(unlikely(!hasColon && *start == ':')){
-//         start++;
-//         hasColon = true;
-//         continue;
-//       }
-//       if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){
-//         kv.value = start;
-//         start++;
-//         continue;
-//       }
-//
-//       if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){
-//         kv.length = start - kv.value;
-//         taosArrayPush(tags, &kv);
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//   }
-//   return tags;
-// }
-
-// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) {
-//   int32_t ret = TSDB_CODE_SUCCESS;
-//
-//   if(is_same_child_table_telnet(elements, &info->preLine) == 0){
-//     return TSDB_CODE_SUCCESS;
-//   }
-//
-//   bool isSameMeasure = IS_SAME_SUPER_TABLE;
-//
-//   int     cnt = 0;
-//   SArray *preLineKV = info->preLineTagKV;
-//   bool    isSuperKVInit = true;
-//   SArray *superKV = NULL;
-//   if(info->dataFormat){
-//     if(unlikely(!isSameMeasure)){
-//       SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//       elements->measureLen, NULL);
-//
-//       if(unlikely(sMeta == NULL)){
-//         sMeta = smlBuildSTableMeta(info->dataFormat);
-//         STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-//         sMeta->tableMeta = pTableMeta;
-//         if(pTableMeta == NULL){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
-//       }
-//       info->currSTableMeta = sMeta->tableMeta;
-//       superKV = sMeta->tags;
-//
-//       if(unlikely(taosArrayGetSize(superKV) == 0)){
-//         isSuperKVInit = false;
-//       }
-//       taosArraySetSize(preLineKV, 0);
-//     }
-//   }else{
-//     taosArraySetSize(preLineKV, 0);
-//   }
-//
-//   SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen);
-//   int32_t tagNum = taosArrayGetSize(tags);
-//   if (tagNum == 0) {
-//     uError("SML:tag is empty:%s", elements->tags)
-//     taosArrayDestroy(tags);
-//     return TSDB_CODE_SML_INVALID_DATA;
-//   }
-//   for (int32_t i = 0; i < tagNum; ++i) {
-//     SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i);
-//
-//     if(info->dataFormat){
-//       if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){
-//         info->dataFormat = false;
-//         info->reRun      = true;
-//         taosArrayDestroy(tags);
-//         return TSDB_CODE_SUCCESS;
-//       }
-//
-//       if(isSameMeasure){
-//         if(unlikely(cnt >= taosArrayGetSize(preLineKV))) {
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-//         if(unlikely(kv.length > preKV->length)){
-//           preKV->length = kv.length;
-//           SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//           elements->measureLen, NULL);
-//            if(unlikely(NULL == tableMeta)){
-//              uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-//              return TSDB_CODE_SML_INTERNAL_ERROR;
-//            }
-//
-//           SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
-//           oldKV->length = kv.length;
-//           info->needModifySchema = true;
-//         }
-//         if(unlikely(!IS_SAME_KEY)){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//       }else{
-//         if(isSuperKVInit){
-//           if(unlikely(cnt >= taosArrayGetSize(superKV))) {
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//           SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-//           if(unlikely(kv.length > preKV->length)) {
-//             preKV->length = kv.length;
-//           }else{
-//             kv.length = preKV->length;
-//           }
-//           info->needModifySchema = true;
-//
-//           if(unlikely(!IS_SAME_KEY)){
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//         }else{
-//           taosArrayPush(superKV, &kv);
-//         }
-//         taosArrayPush(preLineKV, &kv);
-//       }
-//     }else{
-//       taosArrayPush(preLineKV, &kv);
-//     }
-//     cnt++;
-//   }
-//   taosArrayDestroy(tags);
-//
-//   SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES,
-//   is_same_child_table_telnet); if (unlikely(tinfo == NULL)) {
-//     tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-//     if (unlikely(!tinfo)) {
-//       return TSDB_CODE_OUT_OF_MEMORY;
-//     }
-//     tinfo->tags = taosArrayDup(preLineKV, NULL);
-//
-//     smlSetCTableName(tinfo);
-//     if (info->dataFormat) {
-//       info->currSTableMeta->uid = tinfo->uid;
-//       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-//       if (tinfo->tableDataCtx == NULL) {
-//         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-//         return TSDB_CODE_SML_INVALID_DATA;
-//       }
-//     }
-//
-//     SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-//     *key = *elements;
-//     tinfo->key = key;
-//     nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
-//   }
-//   if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
-//
-//   return ret;
-// }
-
 static char *smlJsonGetObj(char *payload) {
   int  leftBracketCnt = 0;
   bool isInQuote = false;
@@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
       break;
     }
     case cJSON_String: {
-      /* set default JSON type to binary/nchar according to
-       * user configured parameter tsDefaultJSONStrType
-       */
-
-      char *tsDefaultJSONStrType = "binary";  // todo
-      smlConvertJSONString(kv, tsDefaultJSONStrType, root);
+      smlConvertJSONString(kv, "binary", root);
       break;
     }
     case cJSON_Object: {
@@ -682,138 +484,70 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
-  int32_t ret = TSDB_CODE_SUCCESS;
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   int32_t tagNum = cJSON_GetArraySize(tags);
   if (unlikely(tagNum == 0)) {
     uError("SML:Tag should not be empty");
-    return TSDB_CODE_TSC_INVALID_JSON;
+    terrno = TSDB_CODE_TSC_INVALID_JSON;
+    return -1;
   }
   for (int32_t i = 0; i < tagNum; ++i) {
     cJSON *tag = cJSON_GetArrayItem(tags, i);
     if (unlikely(tag == NULL)) {
-      return TSDB_CODE_TSC_INVALID_JSON;
+      terrno = TSDB_CODE_TSC_INVALID_JSON;
+      return -1;
     }
-    //    if(unlikely(tag == cMeasure)) continue;
     size_t keyLen = strlen(tag->string);
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
       uError("OTD:Tag key length is 0 or too large than 64");
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno =  TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // add kv to SSmlKv
-    SSmlKv kv = {.key = tag->string, .keyLen = keyLen};
+    SSmlKv kv = {0};
+    kv.key = tag->string;
+    kv.keyLen = keyLen;
+
     // value
-    ret = smlParseValueFromJSON(tag, &kv);
+    int32_t ret = smlParseValueFromJSON(tag, &kv);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      return ret;
-    }
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      terrno =  ret;
+      return -1;
     }
     taosArrayPush(preLineKV, &kv);
+
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno =  TSDB_CODE_SUCCESS;
+      return -1;
+    }
+
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (unlikely(!tinfo)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    if(info->parseJsonByLib){
-    //      key->tags = taosMemoryMalloc(elements->tagsLen + 1);
-    //      memcpy(key->tags, elements->tags, elements->tagsLen);
-    //      key->tags[elements->tagsLen] = 0;
-    //    }
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
-  return ret;
+  ret = smlProcessTagJson(info, tags);
+  if(ret != 0){
+    return terrno;
+  }
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+  return smlProcessChildTable(info, elements);
 }
 
 static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
@@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
@@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-
   if (unlikely(info->lines != NULL)) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
@@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 int32_t smlParseJSON(SSmlHandle *info, char *payload) {

source/client/src/clientSmlLine.c

@@ -20,15 +20,9 @@
 
 #include "clientSml.h"
 
-// comma ,
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-// space
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-// equal =
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
-// quote "
-// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-// SLASH
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
@@ -51,10 +45,10 @@
     }                                                \
   }
 
-#define BINARY_ADD_LEN 2  // "binary"   2 means " "
-#define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
+#define BINARY_ADD_LEN (sizeof("\"\"")-1)    // "binary"   2 means length of ("")
+#define NCHAR_ADD_LEN  (sizeof("L\"\"")-1)   // L"nchar"   3 means length of (L"")
 
-uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
+uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
                                   TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO,
                                   TSDB_TIME_PRECISION_NANO};
 
@@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   return TSDB_CODE_TSC_INVALID_VALUE;
 }
 
-static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
-  if (isSameCTable) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int     cnt = 0;
+static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
-        for (int i = pTableMeta->tableInfo.numOfColumns;
-             i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name,
-                         .keyLen = strlen(tag->name),
-                         .type = tag->type,
-                         .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE};
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
   taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
@@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     while (*sql < sqlEnd) {
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(IS_EQUAL(*sql))) {
         keyLen = *sql - key;
@@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // parse value
@@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         break;
       } else if (unlikely(IS_EQUAL(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(valueLen == 0)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_SML_INVALID_DATA;
+      terrno = TSDB_CODE_SML_INVALID_DATA;
+      return -1;
     }
 
     if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
     if (keyEscaped) {
@@ -331,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       value = tmp;
     }
     SSmlKv kv = {.key = key,
-                 .keyLen = keyLen,
-                 .type = TSDB_DATA_TYPE_NCHAR,
-                 .value = value,
-                 .length = valueLen,
-                 .keyEscaped = keyEscaped,
-                 .valueEscaped = valueEscaped};
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = keyEscaped,
+        .valueEscaped = valueEscaped};
     taosArrayPush(preLineKV, &kv);
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
 
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
     }
 
     cnt++;
@@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
     (*sql)++;
   }
+  return 0;
+}
 
-  void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-  if ((oneTable != NULL)) {
+static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
+  bool isSameCTable = IS_SAME_CHILD_TABLE;
+  if(isSameCTable){
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen);
-  if (unlikely(!tinfo)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tinfo->tags = taosArrayDup(preLineKV, NULL);
-  for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
-    if (kv->keyEscaped) kv->key = NULL;
-    if (kv->valueEscaped) kv->value = NULL;
-  }
-
-  smlSetCTableName(tinfo);
-  getTableUid(info, currElement, tinfo);
-  if (info->dataFormat) {
-    info->currSTableMeta->uid = tinfo->uid;
-    tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-    if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(&tinfo);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-      return TSDB_CODE_SML_INVALID_DATA;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
   }
 
-  taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
+  ret = smlProcessTagLine(info, sql, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
 
-  return TSDB_CODE_SUCCESS;
+  return smlProcessChildTable(info, elements);
 }
 
-static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
+static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
   int cnt = 0;
-  if (info->dataFormat) {
-    if (unlikely(!isSameCTable)) {
-      SSmlTableInfo **oneTable =
-          (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-      if (unlikely(oneTable == NULL)) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-      info->currTableDataCtx = (*oneTable)->tableDataCtx;
-    }
-
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        *tmp = smlBuildSTableMeta(info->dataFormat);
-        if(*tmp == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
-
-        for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
-          if (tag->type == TSDB_DATA_TYPE_NCHAR) {
-            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
-          } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
-            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
-          }
-          taosArrayPush((*tmp)->cols, &kv);
-        }
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxColKVs = (*tmp)->cols;
-    }
-  }
-
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
@@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
 
     if (info->dataFormat) {
-      // cnt begin 0, add ts so + 2
-      if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      // bind data
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1);
-      if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-        uDebug("smlBuildCol error, retry");
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (cnt >= taosArrayGetSize(info->maxColKVs)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
-      if (kv.type != maxKV->type) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      bool isAligned = isSmlColAligned(info, cnt, &kv);
       freeSSmlKv(&kv);
+      if(!isAligned){
+        return TSDB_CODE_SUCCESS;
+      }
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA;
   elements->measure = sql;
-
   // parse measure
   size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
@@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
-
-  bool isSameCTable = false;
-  bool isSameMeasure = false;
-  if (IS_SAME_CHILD_TABLE) {
-    isSameCTable = true;
-    isSameMeasure = true;
-  } else if (info->dataFormat) {
-    isSameMeasure = IS_SAME_SUPER_TABLE;
-  }
+  elements->measureTag = elements->measure;
   // parse tag
   if (*sql == COMMA) sql++;
   elements->tags = sql;
 
-  int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  int ret = smlParseTagLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   elements->cols = sql;
 
-  ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  ret = smlParseColLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  // add ts to
-  SSmlKv kv = {.key = tsSmlTsDefaultName,
-               .keyLen = strlen(tsSmlTsDefaultName),
-               .type = TSDB_DATA_TYPE_TIMESTAMP,
-               .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
-               .keyEscaped = false,
-               .valueEscaped = false};
-  if (info->dataFormat) {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
 
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
-    taosArraySet(elements->colArray, 0, &kv);
-  }
-  info->preLine = *elements;
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  return ret;
+  return smlParseEndLine(info, elements, &kvTs);
 }

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,6 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b) {
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  //  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-  //         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
     return 1;
   return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
@@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
   }
 }
 
-static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
-  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-    elements->measureTag = info->preLine.measureTag;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (!isSameMeasure) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
-  taosArrayClear(preLineKV);
   const char *sql = data;
   while (sql < sqlEnd) {
     JUMP_SPACE(sql, sqlEnd)
@@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     // parse key
     while (sql < sqlEnd) {
       if (unlikely(*sql == SPACE)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(*sql == EQUAL)) {
         keyLen = sql - key;
@@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     }
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
-      smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
-    //    if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) {
-    //      smlBuildInvalidDataMsg(msg, "dumplicate key", key);
-    //      return TSDB_CODE_TSC_DUP_NAMES;
-    //    }
 
     // parse value
     const char *value = sql;
@@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         break;
       }
       if (unlikely(*sql == EQUAL)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       sql++;
     }
     valueLen = sql - value;
 
     if (unlikely(valueLen == 0)) {
-      smlBuildInvalidDataMsg(msg, "invalid value", value);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
+      terrno = TSDB_CODE_TSC_INVALID_VALUE;
+      return -1;
     }
 
     if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-    }
+    SSmlKv kv = {.key = key,
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = false,
+        .valueEscaped = false};
     taosArrayPush(preLineKV, &kv);
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
+    }
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (!tinfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
+static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
+  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
+    elements->measureTag = info->preLine.measureTag;
+    return TSDB_CODE_SUCCESS;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
 
-  return TSDB_CODE_SUCCESS;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
+    }
+  }
+
+  ret = smlProcessTagTelnet(info, data, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+
+  return smlProcessChildTable(info, elements);
 }
 
 // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
@@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
-  bool needConverTime = false;  // get TS before parse tag(get meta), so need conver time
+  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
   if (info->dataFormat && info->currSTableMeta == NULL) {
     needConverTime = true;
   }
@@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
   // parse value
   smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
@@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf);
+  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SUCCESS;
   }
 
-  if (info->dataFormat && info->currSTableMeta != NULL) {
-    if (needConverTime) {
-      kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
-    }
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
+  if (needConverTime) {
+    kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
-  info->preLine = *elements;
 
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }

source/common/src/tdataformat.c

@@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
     n += tPutCStr(p ? p + n : p, pTagVal->pKey);
   } else {
     n += tPutI16v(p ? p + n : p, pTagVal->cid);
+    ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
   }
 
   // type

source/common/src/ttime.c

@@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
     i += 2;
   }
 
+  if (hour > 12 || hour < 0) {
+    return -1;
+  }
+
   // return error if there're illegal charaters after min(2 Digits)
   char* minStr = &str[i];
   if (minStr[1] != '\0' && minStr[2] != '\0') {
@@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
   }
 
   int64_t minute = strnatoi(&str[i], 2);
-  if (minute > 59) {
+  if (minute > 59 || (hour == 12 && minute > 0)) {
     return -1;
   }
 

source/common/src/tvariant.c

@@ -19,6 +19,278 @@
 #include "ttokendef.h"
 #include "tvariant.h"
 
+int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
+  // skip head 0b
+  const char *p = z + 2;
+  int32_t     l = n - 2;
+
+  while (*p == '0' && l > 0) {
+    p++;
+    l--;
+  }
+  if (l > 64) {  // too big
+    return TSDB_CODE_FAILED;
+  }
+
+  uint64_t val = 0;
+  for (int32_t i = 0; i < l; i++) {
+    val = val << 1;
+    if (p[i] == '1') {
+      val |= 1;
+    } else if (p[i] != '0') {  // abnormal char
+      return TSDB_CODE_FAILED;
+    }
+  }
+  *value = val;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) {
+  // parse sign
+  bool has_sign = false;
+  if (z[0] == '-') {
+    *is_neg = true;
+    has_sign = true;
+  } else if (z[0] == '+') {
+    has_sign = true;
+  } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) {
+    return TSDB_CODE_FAILED;
+  }
+  if (has_sign) {
+    if (n < 2) {
+      return TSDB_CODE_FAILED;
+    }
+    z++;
+    n--;
+  }
+
+  errno = 0;
+  char  *endPtr = NULL;
+  bool parsed = false;
+  if (z[0] == '0' && n > 2) {
+    if (z[1] == 'b' || z[1] == 'B') {
+      // paring as binary
+      return parseBinaryUInteger(z, n, value);
+    }
+
+    if (z[1] == 'x' || z[1] == 'X') {
+      // parsing as hex
+      *value = taosStr2UInt64(z, &endPtr, 16);
+      parsed = true;
+    }
+  }
+
+  if (!parsed) {
+    // parsing as double
+    double val = taosStr2Double(z, &endPtr);
+    if (val > UINT64_MAX) {
+      return TSDB_CODE_FAILED;
+    }
+    *value = round(val);
+  }
+
+  if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) {
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  errno = 0;
+  char* endPtr = NULL;
+  *value = taosStr2Double(z, &endPtr);
+
+  if (errno == ERANGE || errno == EINVAL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  switch (type)
+  {
+    case TK_NK_INTEGER: {
+      *value = taosStr2Int64(z, &endPtr, 10);
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(z, &endPtr));
+      if (!IS_VALID_INT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+  default:
+    break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. try to parse as integer
+  *value = taosStr2Int64(z, &endPtr, 10);
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    // pure decimal part
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr+1 < end && endPtr[1] > '4') {
+        const char *p = z;
+        while (*p == ' ') {
+          p++;
+        }
+        if (*p == '-') {
+          if ( *value > INT64_MIN) {
+            (*value)--;
+          }
+        } else {
+          if ( *value < INT64_MAX) {
+            (*value)++;
+          }
+        }
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool     is_neg = false;
+  uint64_t uv = 0;
+  int32_t  code = parseSignAndUInteger(z, n, &is_neg, &uv);
+  if (code == TSDB_CODE_SUCCESS) {
+    // truncate into int64
+    if (is_neg) {
+      if (uv > 1ull + INT64_MAX) {
+        *value = INT64_MIN;
+        return TSDB_CODE_FAILED;
+      } else {
+        *value = -uv;
+      }
+    } else {
+      if (uv > INT64_MAX) {
+        *value = INT64_MAX;
+        return TSDB_CODE_FAILED;
+      }
+      *value = uv;
+    }
+  }
+
+  return code;
+}
+
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  const char *p = z;
+  while (*p == ' ') {
+    p++;
+  }
+  switch (type) {
+    case TK_NK_INTEGER: {
+      *value = taosStr2UInt64(p, &endPtr, 10);
+      if (*p == '-' && *value) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(p, &endPtr));
+      if (!IS_VALID_UINT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+    default:
+      break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. parse as integer
+  *value = taosStr2UInt64(p, &endPtr, 10);
+  if (*p == '-' && *value) {
+    return TSDB_CODE_FAILED;
+  }
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) {
+        (*value)++;
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool    is_neg = false;
+  int32_t code = parseSignAndUInteger(z, n, &is_neg, value);
+  if (is_neg) {
+    if (TSDB_CODE_SUCCESS == code && 0 == *value) {
+      return TSDB_CODE_SUCCESS;
+    }
+    return TSDB_CODE_FAILED;
+  }
+  return code;
+}
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   errno = 0;
   char *endPtr = NULL;
@@ -26,10 +298,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   *value = taosStr2Int64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
@@ -39,16 +311,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
   const char *p = z;
   while (*p == ' ') p++;
   if (*p == '-') {
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
   *value = taosStr2UInt64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
-
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -147,14 +418,13 @@ void taosVariantDestroy(SVariant *pVar) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
-
 }
 
 void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
 
   pDst->nType = pSrc->nType;
-  if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
+  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
       pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON ||
       pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
     int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
@@ -172,7 +442,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i = pSrc->i;
   }
-
 }
 
 int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {

source/common/test/commonTests.cpp

@@ -14,6 +14,7 @@
 #include "tdef.h"
 #include "tvariant.h"
 #include "ttime.h"
+#include "ttokendef.h"
 
 namespace {
 //
@@ -24,40 +25,177 @@ int main(int argc, char** argv) {
   return RUN_ALL_TESTS();
 }
 
-TEST(testCase, toInteger_test) {
+
+TEST(testCase, toUIntegerEx_test) {
+  uint64_t val = 0;
+
   char*    s = "123";
-  uint32_t type = 0;
-
-  int64_t val = 0;
-
-  int32_t ret = toInteger(s, strlen(s), 10, &val);
+  int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 123);
 
+  s = "1000u";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "0b110";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "2567.4787";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2567);
+
+  s = "1.869895343e4";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  s = "-1";
+  ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0b10010";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0x40";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-80.9999";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-5.2343544534e10";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  // INT64_MAX
   s = "9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 9223372036854775807);
 
-  s = "9323372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  // UINT64_MAX
+  s = "18446744073709551615";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 9323372036854775807u);
+  ASSERT_EQ(val, 18446744073709551615u);
+
+  // out of range
+  s = "18446744073709551616";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "5.23e25";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toIntegerEx_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
+  s = "-1";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -1);
+
+  s = "1000u";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "-0x40";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -64);
+
+  s = "0b110";
+  ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "-0b10010";
+  ret = toIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -18);
+
+  s = "-80.9999";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -81);
+
+  s = "2567.8787";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2568);
+
+  s = "-5.2343544534e10";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -52343544534);
+
+  s = "1.869895343e4";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  // INT64_MAX
+  s = "9223372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 9223372036854775807LL);
+
+  s = "-9223372036854775808";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
 
   s = "-9323372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, -1);
 
+  // UINT64_MAX
+  s = "18446744073709551615";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toInteger_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
   s = "-1";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, -1);
 
-  s = "-9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
-  ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, -9223372036854775807);
-
   s = "1000u";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
@@ -77,13 +215,22 @@ TEST(testCase, toInteger_test) {
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 72);
 
-  // 18446744073709551615  UINT64_MAX
-  s = "18446744073709551615";
+  s = "9223372036854775807";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 18446744073709551615u);
+  ASSERT_EQ(val, 9223372036854775807);
 
-  s = "18446744073709551616";
+  s = "-9223372036854775808";
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807"; 
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-9323372036854775807";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
 }

source/dnode/vnode/src/inc/tsdb.h

@@ -681,41 +681,54 @@ struct SDelFWriter {
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
 
-typedef struct STSnapRange STSnapRange;
-typedef TARRAY2(STSnapRange *) TSnapRangeArray;  // disjoint snap ranges
+// fset range
+typedef struct STFileSetRange STFileSetRange;
+typedef TARRAY2(STFileSetRange *) TFileSetRangeArray;  // disjoint ranges
 
-// util
-int32_t   tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
-int32_t   tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
-void      tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap);
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges);
-
-// snap partition list
-typedef TARRAY2(SVersionRange) SVerRangeList;
-typedef struct STsdbSnapPartition STsdbSnapPartition;
-typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList;
-// util
-STsdbSnapPartList *tsdbSnapPartListCreate();
-void               tsdbSnapPartListDestroy(STsdbSnapPartList **ppList);
-int32_t            tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges);
+int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr);
+int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr);
 
+// fset partition
 enum {
-  TSDB_SNAP_RANGE_TYP_HEAD = 0,
-  TSDB_SNAP_RANGE_TYP_DATA,
-  TSDB_SNAP_RANGE_TYP_SMA,
-  TSDB_SNAP_RANGE_TYP_TOMB,
-  TSDB_SNAP_RANGE_TYP_STT,
-  TSDB_SNAP_RANGE_TYP_MAX,
+  TSDB_FSET_RANGE_TYP_HEAD = 0,
+  TSDB_FSET_RANGE_TYP_DATA,
+  TSDB_FSET_RANGE_TYP_SMA,
+  TSDB_FSET_RANGE_TYP_TOMB,
+  TSDB_FSET_RANGE_TYP_STT,
+  TSDB_FSET_RANGE_TYP_MAX,
 };
 
-struct STsdbSnapPartition {
+typedef TARRAY2(SVersionRange) SVerRangeList;
+
+struct STsdbFSetPartition {
   int64_t       fid;
   int8_t        stat;
-  SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX];
+  SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX];
 };
 
+typedef struct STsdbFSetPartition STsdbFSetPartition;
+typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList;
+
+STsdbFSetPartList *tsdbFSetPartListCreate();
+void               tsdbFSetPartListDestroy(STsdbFSetPartList **ppList);
+int32_t            tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges);
+
+// snap rep format
+typedef enum ETsdbRepFmt {
+  TSDB_SNAP_REP_FMT_DEFAULT = 0,
+  TSDB_SNAP_REP_FMT_RAW,
+  TSDB_SNAP_REP_FMT_HYBRID,
+} ETsdbRepFmt;
+
+typedef struct STsdbRepOpts {
+  ETsdbRepFmt format;
+} STsdbRepOpts;
+
+int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+
 // snap read
 struct STsdbReadSnap {
   SMemTable     *pMem;
@@ -1042,7 +1055,7 @@ typedef enum {
 
 // utils
 ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode);
-int32_t      tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap);
+int32_t      tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -65,6 +65,8 @@ typedef struct SMetaSnapReader    SMetaSnapReader;
 typedef struct SMetaSnapWriter    SMetaSnapWriter;
 typedef struct STsdbSnapReader    STsdbSnapReader;
 typedef struct STsdbSnapWriter    STsdbSnapWriter;
+typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
+typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
 typedef struct STqSnapReader      STqSnapReader;
 typedef struct STqSnapWriter      STqSnapWriter;
 typedef struct STqOffsetReader    STqOffsetReader;
@@ -313,6 +315,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr);
 int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter);
 int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapRAWReader ========================================
+int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData);
+// STsdbSnapRAWWriter ========================================
+int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter);
+int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
+int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
+int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
 // STqSnapshotReader ==
 int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader);
 int32_t tqSnapReaderClose(STqSnapReader** ppReader);
@@ -531,6 +542,7 @@ enum {
   SNAP_DATA_STREAM_STATE = 11,
   SNAP_DATA_STREAM_STATE_BACKEND = 12,
   SNAP_DATA_TQ_CHECKINFO = 13,
+  SNAP_DATA_RAW = 14,
 };
 
 struct SSnapDataHdr {

source/dnode/vnode/src/tq/tqUtil.c

@@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
     taosWUnLockLatch(&pTq->lock);
   }
 
+  dataRsp.reqOffset = *pOffset;   // reqOffset represents the current date offset, may be changed if wal not exists
   code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
 
 end : {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -1167,26 +1167,33 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
 
     // taosThreadMutexLock(&pTsdb->lruMutex);
 
+    bool       erase = false;
     LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen);
     if (h) {
       SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h);
-      if (pLastCol->dirty) {
+      if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
         pLastCol->dirty = 0;
+        erase = true;
       }
-      taosLRUCacheRelease(pTsdb->lruCache, h, true);
+      taosLRUCacheRelease(pTsdb->lruCache, h, erase);
+    }
+    if (erase) {
+      taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
     }
-    taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
 
+    erase = false;
     h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen);
     if (h) {
       SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h);
-      if (pLastCol->dirty) {
+      if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
         pLastCol->dirty = 0;
+        erase = true;
       }
-      taosLRUCacheRelease(pTsdb->lruCache, h, true);
+      taosLRUCacheRelease(pTsdb->lruCache, h, erase);
+    }
+    if (erase) {
+      taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
     }
-    taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
-
     // taosThreadMutexUnlock(&pTsdb->lruMutex);
   }
   for (int i = 0; i < num_keys; ++i) {
@@ -1797,8 +1804,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
       }
 
       if (record.version <= pReader->info.verRange.maxVer) {
-        tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
-                  TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
+        /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
+          TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/
 
         SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
         taosArrayPush(pInfo->pTombData, &delData);

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -22,19 +22,34 @@
 
 #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
 
+static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
+  char *buf = taosMemoryCalloc(1, pCol->info.bytes);
+  SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
+  pRes->bytes = 0;
+  pRes->hasResult = true;
+  pRes->isNull = true;
+  varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE);
+  colDataSetVal(pCol, row, buf, false);
+  taosMemoryFree(buf);
+}
+
 static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds,
                           const int32_t* dstSlotIds, void** pRes, const char* idStr) {
   int32_t numOfRows = pBlock->info.rows;
   // bool    allNullRow = true;
 
   if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
-    uint64_t ts = 0;
-    SFirstLastRes* p;
-    col_id_t colId;
+    uint64_t ts = TSKEY_MIN;
+    SFirstLastRes* p = NULL;
+    col_id_t colId = -1;
     for (int32_t i = 0; i < pReader->numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
-      int32_t          slotId = slotIds[i];
-      SLastCol*        pColVal = (SLastCol*)taosArrayGet(pRow, i);
+      if (slotIds[i] == -1) {
+        setFirstLastResColToNull(pColInfoData, numOfRows);
+        continue;
+      }
+      int32_t   slotId = slotIds[i];
+      SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       colId = pColVal->colVal.cid;
       p = (SFirstLastRes*)varDataVal(pRes[i]);
 
@@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
     for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
       SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
       if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
-        colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        if (ts == TSKEY_MIN) {
+          colDataSetNULL(pCol, numOfRows);
+        } else {
+          colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        }
         continue;
-      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) {
-        if (!p->isNull) {
+      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) {
+        if (p && !p->isNull) {
           colDataSetVal(pCol, numOfRows, p->buf, false);
         } else {
           colDataSetNULL(pCol, numOfRows);
@@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
 
       int32_t   slotId = slotIds[i];
+      if (slotId == -1) {
+        colDataSetNULL(pColInfoData, numOfRows);
+        continue;
+      }
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       SColVal*  pVal = &pColVal->colVal;
 
@@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   }
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
-    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE);
+    int32_t bytes;
+    if (slotIds[j] == -1)
+      bytes = 1;
+    else
+      bytes = pr->pSchema->columns[slotIds[j]].bytes;
+
+    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE);
     SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
     p->ts = INT64_MIN;
   }
@@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
     for (int32_t i = 0; i < pr->numOfCols; ++i) {
       int32_t          slotId = slotIds[i];
+      if (slotId == -1) {
+        SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL};
+        taosArrayPush(pLastCols, &p);
+        continue;
+      }
       struct STColumn* pCol = &pr->pSchema->columns[slotId];
       SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
 
@@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         bool    hasNotNullRow = true;
         int64_t singleTableLastTs = INT64_MAX;
         for (int32_t k = 0; k < pr->numOfCols; ++k) {
+          if (slotIds[k] == -1) continue;
           SLastCol* p = taosArrayGet(pLastCols, k);
           SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
 

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -435,7 +435,7 @@ _exit:
     tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
               __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
   }
-  return 0;
+  return code;
 }
 
 static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDataFileRAW.h"
+
+// SDataFileRAWReader =============================================
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader));
+  if (reader[0] == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  reader[0]->config[0] = config[0];
+
+  if (fname) {
+    if (fname) {
+      code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else {
+    char fname1[TSDB_FILENAME_LEN];
+    tsdbTFileName(config->tsdb, &config->file, fname1);
+    code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) {
+  if (reader[0] == NULL) return 0;
+
+  if (reader[0]->fd) {
+    tsdbCloseFile(&reader[0]->fd);
+  }
+
+  taosMemoryFree(reader[0]);
+  reader[0] = NULL;
+  return 0;
+}
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  pBlock->file.type = reader->config->file.type;
+  pBlock->file.fid = reader->config->file.fid;
+  pBlock->file.cid = reader->config->file.cid;
+  pBlock->file.size = reader->config->file.size;
+  pBlock->file.minVer = reader->config->file.minVer;
+  pBlock->file.maxVer = reader->config->file.maxVer;
+  pBlock->file.stt->level = reader->config->file.stt->level;
+
+  code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+// SDataFileRAWWriter =============================================
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter));
+  if (!writer) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer->config[0] = config[0];
+
+  code = tsdbDataFileRAWWriterDoOpen(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    taosMemoryFree(writer);
+    writer = NULL;
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  ppWriter[0] = writer;
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
+  ASSERT(0);
+  return 0;
+}
+
+static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
+
+static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
+  int32_t  code = 0;
+  int32_t  lino = 0;
+  ASSERT(writer->ctx->offset == writer->file.size);
+  ASSERT(writer->config->fid == writer->file.fid);
+
+  STFileOp op = (STFileOp){
+      .optype = TSDB_FOP_CREATE,
+      .fid = writer->config->fid,
+      .nf = writer->file,
+  };
+  code = TARRAY2_APPEND(opArr, op);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  if (writer->fd) {
+    code = tsdbFsyncFile(writer->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+    tsdbCloseFile(&writer->fd);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  char    fname[TSDB_FILENAME_LEN];
+  int32_t flag = TD_FILE_READ | TD_FILE_WRITE;
+
+  if (writer->ctx->offset == 0) {
+    flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
+  }
+
+  tsdbTFileName(writer->config->tsdb, &writer->file, fname);
+  code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  writer->file = writer->config->file;
+  writer->ctx->offset = 0;
+
+  code = tsdbDataFileRAWWriterOpenDataFD(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->opened = true;
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  if (writer[0]->ctx->opened) {
+    if (abort) {
+      code = tsdbDataFileRAWWriterCloseAbort(writer[0]);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    } else {
+      code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    tsdbDataFileRAWWriterDoClose(writer[0]);
+  }
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->offset += pDataBlock->dataLength;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tarray2.h"
+#include "tsdbDef.h"
+#include "tsdbFSet2.h"
+#include "tsdbFile2.h"
+#include "tsdbUtil2.h"
+
+#ifndef _TSDB_DATA_FILE_RAW_H
+#define _TSDB_DATA_FILE_RAW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// STsdbDataRAWBlockHeader =======================================
+typedef struct STsdbDataRAWBlockHeader {
+  struct {
+    int32_t type;
+    int64_t fid;
+    int64_t cid;
+    int64_t size;
+    int64_t minVer;
+    int64_t maxVer;
+    union {
+      struct {
+        int32_t level;
+      } stt[1];
+    };
+  } file;
+
+  int64_t offset;
+  int64_t dataLength;
+  uint8_t data[0];
+} STsdbDataRAWBlockHeader;
+
+// SDataFileRAWReader =============================================
+typedef struct SDataFileRAWReaderConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  STFile file;
+} SDataFileRAWReaderConfig;
+
+typedef struct SDataFileRAWReader {
+  SDataFileRAWReaderConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STsdbFD *fd;
+} SDataFileRAWReader;
+
+typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray;
+
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader);
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader);
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr);
+
+// SDataFileRAWWriter =============================================
+typedef struct SDataFileRAWWriterConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  SDiskID did;
+  int64_t fid;
+  int64_t cid;
+  int32_t level;
+
+  STFile file;
+} SDataFileRAWWriterConfig;
+
+typedef struct SDataFileRAWWriter {
+  SDataFileRAWWriterConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STFile   file;
+  STsdbFD *fd;
+} SDataFileRAWWriter;
+
+typedef struct SDataFileRAWWriter SDataFileRAWWriter;
+
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer);
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr);
+int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_DATA_FILE_RAW_H*/

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -1072,7 +1072,25 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) {
   return 0;
 }
 
-int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr,
+static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) {
+  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
+  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+  if (pHash == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
+    STFileSetRange *u = TARRAY2_GET(pRanges, i);
+    int32_t         fid = u->fid;
+    int32_t         code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
+    ASSERT(code == 0);
+    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
+  }
+  return pHash;
+}
+
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr,
                                        TFileOpArray *fopArr) {
   int32_t    code = 0;
   STFileSet *fset;
@@ -1084,7 +1102,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
   TARRAY2_INIT(fsetArr[0]);
 
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1096,7 +1114,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
     int64_t ever = VERSION_MAX;
     if (pHash) {
       int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         ever = u->sver - 1;
       }
@@ -1123,29 +1141,13 @@ _out:
   return code;
 }
 
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) {
-  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
-  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
-  if (pHash == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); }
 
-  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
-    STSnapRange *u = TARRAY2_GET(pRanges, i);
-    int32_t      fid = u->fid;
-    int32_t      code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
-    ASSERT(code == 0);
-    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
-  }
-  return pHash;
-}
-
-int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
-                                      TSnapRangeArray **fsrArr) {
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges,
+                                      TFileSetRangeArray **fsrArr) {
   int32_t      code = 0;
   STFileSet   *fset;
-  STSnapRange *fsr1 = NULL;
+  STFileSetRange *fsr1 = NULL;
   SHashObj    *pHash = NULL;
 
   fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0]));
@@ -1156,7 +1158,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
   tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges)));
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1170,7 +1172,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     if (pHash) {
       int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         sver1 = u->sver;
         tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
@@ -1184,7 +1186,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1);
 
-    code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
+    code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
     if (code) break;
 
     code = TARRAY2_APPEND(fsrArr[0], fsr1);
@@ -1195,8 +1197,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
   taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   if (code) {
-    tsdbTSnapRangeClear(&fsr1);
-    TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear);
+    tsdbTFileSetRangeClear(&fsr1);
+    TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear);
     fsrArr[0] = NULL;
   }
 
@@ -1206,4 +1208,6 @@ _out:
     pHash = NULL;
   }
   return code;
-}
+}
+
+int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }

source/dnode/vnode/src/tsdb/tsdbFS2.h

@@ -44,13 +44,13 @@ int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSCreateRefSnapshotWithoutLock(STFileSystem *fs, TFileSetArray **fsetArr);
 int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr);
 
-int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pExclude, TFileSetArray **fsetArr,
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pExclude, TFileSetArray **fsetArr,
                                        TFileOpArray *fopArr);
-int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr, TFileOpArray *fopArr);
-int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
-                                      TSnapRangeArray **fsrArr);
-int32_t tsdbFSDestroyRefRangedSnapshot(TSnapRangeArray **fsrArr);
-// txn
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr);
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges,
+                                      TFileSetRangeArray **fsrArr);
+int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr);
+//  txn
 int64_t tsdbFSAllocEid(STFileSystem *fs);
 int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype);
 int32_t tsdbFSEditCommit(STFileSystem *fs);

source/dnode/vnode/src/tsdb/tsdbFSet2.c

@@ -533,7 +533,8 @@ int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_
   return 0;
 }
 
-int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr) {
+int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever,
+                                 STFileSetRange **fsr) {
   fsr[0] = taosMemoryCalloc(1, sizeof(*fsr[0]));
   if (fsr[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
   fsr[0]->fid = fset1->fid;
@@ -575,7 +576,7 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs
   return 0;
 }
 
-int32_t tsdbTSnapRangeClear(STSnapRange **fsr) {
+int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr) {
   if (!fsr[0]) return 0;
 
   tsdbTFileSetClear(&fsr[0]->fset);
@@ -584,6 +585,15 @@ int32_t tsdbTSnapRangeClear(STSnapRange **fsr) {
   return 0;
 }
 
+int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray** ppArr) {
+  if (ppArr && ppArr[0]) {
+    TARRAY2_DESTROY(ppArr[0], tsdbTFileSetRangeClear);
+    taosMemoryFree(ppArr[0]);
+    ppArr[0] = NULL;
+  }
+  return 0;
+}
+
 int32_t tsdbTFileSetClear(STFileSet **fset) {
   if (!fset[0]) return 0;
 

source/dnode/vnode/src/tsdb/tsdbFSet2.h

@@ -49,8 +49,8 @@ int32_t tsdbTFileSetRemove(STFileSet *fset);
 int32_t tsdbTFileSetFilteredInitDup(STsdb *pTsdb, const STFileSet *fset1, int64_t ever, STFileSet **fset,
                                     TFileOpArray *fopArr);
 
-int32_t tsdbTSnapRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever, STSnapRange **fsr);
-int32_t tsdbTSnapRangeClear(STSnapRange **fsr);
+int32_t tsdbTFileSetRangeInitRef(STsdb *pTsdb, const STFileSet *fset1, int64_t sver, int64_t ever,
+                                 STFileSetRange **fsr);
 
 // to/from json
 int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json);
@@ -101,7 +101,7 @@ struct STFileSet {
   bool         blockCommit;
 };
 
-struct STSnapRange {
+struct STFileSetRange {
   int32_t    fid;
   int64_t    sver;
   int64_t    ever;

source/dnode/vnode/src/tsdb/tsdbFSetRAW.c

@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbFSetRAW.h"
+
+// SFSetRAWWriter ==================================================
+typedef struct SFSetRAWWriter {
+  SFSetRAWWriterConfig config[1];
+
+  struct {
+    TFileOpArray fopArr[1];
+    STFile       file;
+    int64_t      offset;
+  } ctx[1];
+
+  // writer
+  SDataFileRAWWriter *dataWriter;
+} SFSetRAWWriter;
+
+int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));
+  if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer[0]->config[0] = config[0];
+
+  TARRAY2_INIT(writer[0]->ctx->fopArr);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb *tsdb = writer->config->tsdb;
+
+  STFileOp op;
+  TARRAY2_FOREACH(writer->ctx->fopArr, op) {
+    code = TARRAY2_APPEND(fopArr, op);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  TARRAY2_CLEAR(writer->ctx->fopArr, NULL);
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SDataFileRAWWriterConfig config = {
+      .tsdb = writer->config->tsdb,
+      .szPage = writer->config->szPage,
+      .fid = bHdr->file.fid,
+      .did = writer->config->did,
+      .cid = writer->config->cid,
+      .level = writer->config->level,
+
+      .file =
+          {
+              .type = bHdr->file.type,
+              .fid = bHdr->file.fid,
+              .did = writer->config->did,
+              .cid = writer->config->cid,
+              .size = bHdr->file.size,
+              .minVer = bHdr->file.minVer,
+              .maxVer = bHdr->file.maxVer,
+              .stt = {{
+                  .level = bHdr->file.stt->level,
+              }},
+          },
+  };
+
+  writer->ctx->offset = 0;
+  writer->ctx->file = config.file;
+
+  code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb *tsdb = writer[0]->config->tsdb;
+
+  // end
+  code = tsdbFSetRAWWriteFileDataEnd(writer[0]);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbFSetRAWWriterFinish(writer[0], fopArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+  // free
+  TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL);
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(writer->ctx->offset >= 0 && writer->ctx->offset <= writer->ctx->file.size);
+
+  if (writer->ctx->offset == writer->ctx->file.size) {
+    code = tsdbFSetRAWWriteFileDataEnd(writer);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->offset += bHdr->dataLength;
+  ASSERT(writer->ctx->offset == writer->dataWriter->ctx->offset);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbFSetRAW.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDataFileRAW.h"
+
+#ifndef _TSDB_FSET_RAW_H
+#define _TSDB_FSET_RAW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct SFSetRAWWriterConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  SDiskID did;
+  int64_t fid;
+  int64_t cid;
+  int32_t level;
+} SFSetRAWWriterConfig;
+
+typedef struct SFSetRAWWriter SFSetRAWWriter;
+
+int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer);
+int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr);
+int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_FSET_RAW_H*/

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -47,6 +47,8 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan
                                   STsdbReader* pReader, SRow** pTSRow);
 static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key,
                                      STsdbReader* pReader);
+static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo,
+                                    STsdbReader* pReader);
 
 static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);
 static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
@@ -1746,6 +1748,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader*
                                          STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
   SRowMerger*         pMerger = &pReader->status.merger;
+  int32_t             code = TSDB_CODE_SUCCESS;
 
   // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized
   if (pMerger->pArray == NULL) {
@@ -1756,45 +1759,68 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader*
     }
   }
 
-  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {
-    // no last block available, only data block exists
-    if (!hasDataInSttBlock(pSttBlockReader)) {
-      return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
-    }
-
-    // row in last file block
+  bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo);
+  bool dataInSttFile = hasDataInSttBlock(pSttBlockReader);
+  if (dataInDataFile && (!dataInSttFile)) {
+    // no stt file block available, only data block exists
+    return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
+  } else if ((!dataInDataFile) && dataInSttFile) {
+    // no data ile block exists
+    return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
+  } else {
+    // row in both stt file blocks and data file blocks
     TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
     int64_t tsLast = getCurrentKeyInSttBlock(pSttBlockReader);
     if (ASCENDING_TRAVERSE(pReader->info.order)) {
-      if (key < tsLast) {
+      if (key < tsLast) {  // asc
         return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
       } else if (key > tsLast) {
-        return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false);
+        return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
       }
-    } else {
+    } else {  // desc
       if (key > tsLast) {
         return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader);
       } else if (key < tsLast) {
-        return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false);
+        return mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
       }
     }
+
     // the following for key == tsLast
+    // ASC:  file block ------> stt  block
+    // DESC: stt  block ------> file block
     SRow*   pTSRow = NULL;
-    int32_t code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
+    if (ASCENDING_TRAVERSE(pReader->info.order)) {
+      code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader);
+
+      TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
+      code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr);
+    } else {
+      TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
+      code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr);
+
+      code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+
+      doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader);
     }
 
-    doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader);
-
-    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
-    code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
-    if (code != TSDB_CODE_SUCCESS) {
-      return code;
-    }
-
-    doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr);
-
     code = tsdbRowMergerGetRow(pMerger, &pTSRow);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
@@ -1805,9 +1831,6 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader*
     taosMemoryFree(pTSRow);
     tsdbRowMergerClear(pMerger);
     return code;
-
-  } else {  // only last block exists
-    return doMergeFileBlockAndLastBlock(pSttBlockReader, pReader, pBlockScanInfo, NULL, false);
   }
 }
 
@@ -1894,8 +1917,8 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
     }
   }
 
-  // ASC: file block -----> last block -----> imem -----> mem
-  // DESC: mem -----> imem -----> last block -----> file block
+  // ASC: file block -----> stt block -----> imem -----> mem
+  // DESC: mem -----> imem -----> stt block -----> file block
   if (ASCENDING_TRAVERSE(pReader->info.order)) {
     if (minKey == key) {
       TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
@@ -2205,6 +2228,50 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc
   }
 }
 
+int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
+  bool     copied = false;
+  SRow*    pTSRow = NULL;
+  int64_t  tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader);
+
+  SRowMerger* pMerger = &pReader->status.merger;
+  TSDBROW*    pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
+  TSDBROW     fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
+
+  tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid,
+            fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr);
+
+  int32_t code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied);
+  if (code) {
+    return code;
+  }
+
+  if (copied) {
+    pBlockScanInfo->lastProcKey = tsLastBlock;
+    return TSDB_CODE_SUCCESS;
+  } else {
+    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
+    tsdbRowMergerAdd(pMerger, pRow1, NULL);
+    doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange,
+                          pReader->idStr);
+
+    code = tsdbRowMergerGetRow(pMerger, &pTSRow);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
+
+    taosMemoryFree(pTSRow);
+    tsdbRowMergerClear(pMerger);
+    return code;
+  }
+}
+
 static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo,
                                           SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) {
   SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo;
@@ -2226,17 +2293,17 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI
     return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader);
   }
 
-  // imem + file + last block
+  // imem + file + stt block
   if (pBlockScanInfo->iiter.hasVal) {
     return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pSttBlockReader);
   }
 
-  // mem + file + last block
+  // mem + file + stt block
   if (pBlockScanInfo->iter.hasVal) {
     return doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, key, pSttBlockReader);
   }
 
-  // files data blocks + last block
+  // files data blocks + stt block
   return mergeFileBlockAndSttBlock(pReader, pSttBlockReader, key, pBlockScanInfo, pBlockData);
 }
 

source/dnode/vnode/src/tsdb/tsdbSnapInfo.c

@@ -0,0 +1,627 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+#include "tsdbFS2.h"
+
+#define TSDB_SNAP_MSG_VER 1
+
+// fset partition
+static int32_t tsdbFSetPartCmprFn(STsdbFSetPartition* x, STsdbFSetPartition* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) {
+  if (x->minVer < y->minVer) return -1;
+  if (x->minVer > y->minVer) return 1;
+  if (x->maxVer < y->maxVer) return -1;
+  if (x->maxVer > y->maxVer) return 1;
+  return 0;
+}
+
+static int32_t tsdbTFileSetRangeCmprFn(STFileSetRange* x, STFileSetRange* y) {
+  if (x->fid < y->fid) return -1;
+  if (x->fid > y->fid) return 1;
+  return 0;
+}
+
+STsdbFSetPartition* tsdbFSetPartitionCreate() {
+  STsdbFSetPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbFSetPartition));
+  if (pSP == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) {
+    TARRAY2_INIT(&pSP->verRanges[i]);
+  }
+  return pSP;
+}
+
+void tsdbFSetPartitionClear(STsdbFSetPartition** ppSP) {
+  if (ppSP == NULL || ppSP[0] == NULL) {
+    return;
+  }
+  for (int32_t i = 0; i < TSDB_FSET_RANGE_TYP_MAX; i++) {
+    TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL);
+  }
+  taosMemoryFree(ppSP[0]);
+  ppSP[0] = NULL;
+}
+
+static int32_t tsdbFTypeToFRangeType(tsdb_ftype_t ftype) {
+  switch (ftype) {
+    case TSDB_FTYPE_HEAD:
+      return TSDB_FSET_RANGE_TYP_HEAD;
+    case TSDB_FTYPE_DATA:
+      return TSDB_FSET_RANGE_TYP_DATA;
+    case TSDB_FTYPE_SMA:
+      return TSDB_FSET_RANGE_TYP_SMA;
+    case TSDB_FTYPE_TOMB:
+      return TSDB_FSET_RANGE_TYP_TOMB;
+    case TSDB_FTYPE_STT:
+      return TSDB_FSET_RANGE_TYP_STT;
+  }
+  return TSDB_FSET_RANGE_TYP_MAX;
+}
+
+static int32_t tsdbTFileSetToFSetPartition(STFileSet* fset, STsdbFSetPartition** ppSP) {
+  STsdbFSetPartition* p = tsdbFSetPartitionCreate();
+  if (p == NULL) {
+    goto _err;
+  }
+
+  p->fid = fset->fid;
+
+  int32_t code = 0;
+  int32_t typ = 0;
+  int32_t corrupt = false;
+  int32_t count = 0;
+  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
+    if (fset->farr[ftype] == NULL) continue;
+    typ = tsdbFTypeToFRangeType(ftype);
+    ASSERT(typ < TSDB_FSET_RANGE_TYP_MAX);
+    STFile* f = fset->farr[ftype]->f;
+    if (f->maxVer > fset->maxVerValid) {
+      corrupt = true;
+      tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                ", ftype: %d",
+                fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype);
+      continue;
+    }
+    count++;
+    SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+
+  typ = TSDB_FSET_RANGE_TYP_STT;
+  const SSttLvl* lvl;
+  TARRAY2_FOREACH(fset->lvlArr, lvl) {
+    STFileObj* fobj;
+    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+      STFile* f = fobj->f;
+      if (f->maxVer > fset->maxVerValid) {
+        corrupt = true;
+        tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
+                  ", ftype: %d",
+                  fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ);
+        continue;
+      }
+      count++;
+      SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
+      code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+      ASSERT(code == 0);
+    }
+  }
+  if (corrupt && count == 0) {
+    SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid};
+    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppSP[0] = p;
+  return 0;
+
+_err:
+  tsdbFSetPartitionClear(&p);
+  return -1;
+}
+
+// fset partition list
+STsdbFSetPartList* tsdbFSetPartListCreate() {
+  STsdbFSetPartList* pList = taosMemoryCalloc(1, sizeof(STsdbFSetPartList));
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  TARRAY2_INIT(pList);
+  return pList;
+}
+
+void tsdbFSetPartListDestroy(STsdbFSetPartList** ppList) {
+  if (ppList == NULL || ppList[0] == NULL) return;
+
+  TARRAY2_DESTROY(ppList[0], tsdbFSetPartitionClear);
+  taosMemoryFree(ppList[0]);
+  ppList[0] = NULL;
+}
+
+int32_t tsdbFSetPartListToRangeDiff(STsdbFSetPartList* pList, TFileSetRangeArray** ppRanges) {
+  TFileSetRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TFileSetRangeArray));
+  if (pDiff == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  TARRAY2_INIT(pDiff);
+
+  STsdbFSetPartition* part;
+  TARRAY2_FOREACH(pList, part) {
+    STFileSetRange* r = taosMemoryCalloc(1, sizeof(STFileSetRange));
+    if (r == NULL) {
+      terrno = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+    int64_t maxVerValid = -1;
+    int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &part->verRanges[i];
+      SVersionRange  vr = {0};
+      TARRAY2_FOREACH(iList, vr) {
+        if (vr.maxVer < vr.minVer) {
+          continue;
+        }
+        maxVerValid = TMAX(maxVerValid, vr.maxVer);
+      }
+    }
+    r->fid = part->fid;
+    r->sver = maxVerValid + 1;
+    r->ever = VERSION_MAX;
+    tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever);
+    int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbTFileSetRangeCmprFn);
+    ASSERT(code == 0);
+  }
+  ppRanges[0] = pDiff;
+
+  tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff));
+  return 0;
+
+_err:
+  if (pDiff) {
+    tsdbTFileSetRangeArrayDestroy(&pDiff);
+  }
+  return -1;
+}
+
+// serialization
+int32_t tTsdbFSetPartListDataLenCalc(STsdbFSetPartList* pList) {
+  int32_t hdrLen = sizeof(int32_t);
+  int32_t datLen = 0;
+
+  int8_t  msgVer = 1;
+  int32_t len = TARRAY2_SIZE(pList);
+  hdrLen += sizeof(msgVer);
+  hdrLen += sizeof(len);
+  datLen += hdrLen;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbFSetPartition* p = TARRAY2_GET(pList, u);
+    int32_t             typMax = TSDB_FSET_RANGE_TYP_MAX;
+    int32_t             uItem = 0;
+    uItem += sizeof(STsdbFSetPartition);
+    uItem += sizeof(typMax);
+
+    for (int32_t i = 0; i < typMax; i++) {
+      int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]);
+      int32_t jItem = 0;
+      jItem += sizeof(SVersionRange);
+      jItem += sizeof(int64_t);
+      uItem += sizeof(iLen) + jItem * iLen;
+    }
+    datLen += uItem;
+  }
+  return datLen;
+}
+
+int32_t tSerializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  int8_t  msgVer = TSDB_SNAP_MSG_VER;
+  int32_t len = TARRAY2_SIZE(pList);
+
+  if (tStartEncode(&encoder) < 0) goto _err;
+  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
+  if (tEncodeI32(&encoder, len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    STsdbFSetPartition* p = TARRAY2_GET(pList, u);
+    if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
+    if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
+    if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
+    if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
+
+    int32_t typMax = TSDB_FSET_RANGE_TYP_MAX;
+    if (tEncodeI32(&encoder, typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = TARRAY2_SIZE(iList);
+
+      if (tEncodeI32(&encoder, iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = TARRAY2_GET(iList, j);
+        if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
+        if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
+      }
+    }
+  }
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+
+_err:
+  tEncoderClear(&encoder);
+  return -1;
+}
+
+int32_t tDeserializeTsdbFSetPartList(void* buf, int32_t bufLen, STsdbFSetPartList* pList) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  int8_t  reserved8 = 0;
+  int16_t reserved16 = 0;
+  int64_t reserved64 = 0;
+
+  STsdbFSetPartition* p = NULL;
+
+  int8_t  msgVer = 0;
+  int32_t len = 0;
+  if (tStartDecode(&decoder) < 0) goto _err;
+  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
+  if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
+  if (tDecodeI32(&decoder, &len) < 0) goto _err;
+
+  for (int32_t u = 0; u < len; u++) {
+    p = tsdbFSetPartitionCreate();
+    if (p == NULL) goto _err;
+    if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
+    if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
+    if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
+    if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
+
+    int32_t typMax = 0;
+    if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
+
+    for (int32_t i = 0; i < typMax; i++) {
+      SVerRangeList* iList = &p->verRanges[i];
+      int32_t        iLen = 0;
+      if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
+      for (int32_t j = 0; j < iLen; j++) {
+        SVersionRange r = {0};
+        if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
+        if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
+        TARRAY2_APPEND(iList, r);
+      }
+    }
+    TARRAY2_APPEND(pList, p);
+    p = NULL;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+
+_err:
+  if (p) {
+    tsdbFSetPartitionClear(&p);
+  }
+  tDecoderClear(&decoder);
+  return -1;
+}
+
+// fs state
+static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
+  STsdbFSetPartList* pList = tsdbFSetPartListCreate();
+  if (pList == NULL) {
+    return NULL;
+  }
+
+  int32_t code = 0;
+  taosThreadMutexLock(&fs->tsdb->mutex);
+  STFileSet* fset;
+  TARRAY2_FOREACH(fs->fSetArr, fset) {
+    STsdbFSetPartition* pItem = NULL;
+    if (tsdbTFileSetToFSetPartition(fset, &pItem) < 0) {
+      code = -1;
+      break;
+    }
+    ASSERT(pItem != NULL);
+    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn);
+    ASSERT(code == 0);
+  }
+  taosThreadMutexUnlock(&fs->tsdb->mutex);
+
+  if (code) {
+    TARRAY2_DESTROY(pList, tsdbFSetPartitionClear);
+    taosMemoryFree(pList);
+    pList = NULL;
+  }
+  return pList;
+}
+
+ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
+  if (!VND_IS_RSMA(pVnode)) {
+    return pVnode->pTsdb->pFS->fsstate;
+  }
+  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
+    if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
+      return TSDB_FS_STATE_INCOMPLETE;
+    }
+  }
+  return TSDB_FS_STATE_NORMAL;
+}
+
+// description
+typedef struct STsdbPartitionInfo {
+  int32_t            vgId;
+  int32_t            tsdbMaxCnt;
+  int32_t            subTyps[TSDB_RETENTION_MAX];
+  STsdbFSetPartList* pLists[TSDB_RETENTION_MAX];
+} STsdbPartitionInfo;
+
+static int32_t tsdbPartitionInfoInit(SVnode* pVnode, STsdbPartitionInfo* pInfo) {
+  int32_t subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
+  pInfo->vgId = TD_VID(pVnode);
+  pInfo->tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX);
+
+  ASSERT(sizeof(pInfo->subTyps) == sizeof(subTyps));
+  memcpy(pInfo->subTyps, (char*)subTyps, sizeof(subTyps));
+
+  // fset partition list
+  memset(pInfo->pLists, 0, sizeof(pInfo->pLists[0]) * TSDB_RETENTION_MAX);
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
+    pInfo->pLists[j] = tsdbSnapGetFSetPartList(pTsdb->pFS);
+    if (pInfo->pLists[j] == NULL) return -1;
+  }
+  return 0;
+}
+
+static void tsdbPartitionInfoClear(STsdbPartitionInfo* pInfo) {
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    if (pInfo->pLists[j] == NULL) continue;
+    tsdbFSetPartListDestroy(&pInfo->pLists[j]);
+  }
+}
+
+static int32_t tsdbPartitionInfoEstSize(STsdbPartitionInfo* pInfo) {
+  int32_t dataLen = 0;
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    dataLen += sizeof(SSyncTLV);  // subTyps[j]
+    dataLen += tTsdbFSetPartListDataLenCalc(pInfo->pLists[j]);
+  }
+  return dataLen;
+}
+
+static int32_t tsdbPartitionInfoSerialize(STsdbPartitionInfo* pInfo, uint8_t* buf, int32_t bufLen) {
+  int32_t tlen = 0;
+  int32_t offset = 0;
+  for (int32_t j = 0; j < pInfo->tsdbMaxCnt; ++j) {
+    SSyncTLV* pSubHead = (void*)((char*)buf + offset);
+    int32_t   valOffset = offset + sizeof(*pSubHead);
+    ASSERT(pSubHead->val == (char*)buf + valOffset);
+    if ((tlen = tSerializeTsdbFSetPartList(pSubHead->val, bufLen - valOffset, pInfo->pLists[j])) < 0) {
+      tsdbError("vgId:%d, failed to serialize fset partition list of tsdb %d since %s", pInfo->vgId, j, terrstr());
+      return -1;
+    }
+    pSubHead->typ = pInfo->subTyps[j];
+    pSubHead->len = tlen;
+    offset += sizeof(*pSubHead) + tlen;
+  }
+  return offset;
+}
+
+// tsdb replication opts
+static int32_t tTsdbRepOptsDataLenCalc(STsdbRepOpts* pInfo) {
+  int32_t hdrLen = sizeof(int32_t);
+  int32_t datLen = 0;
+
+  int8_t  msgVer = 0;
+  int64_t reserved64 = 0;
+  int16_t format = 0;
+  hdrLen += sizeof(msgVer);
+  datLen += hdrLen;
+  datLen += sizeof(format);
+  datLen += sizeof(reserved64);
+  datLen += sizeof(*pInfo);
+  return datLen;
+}
+
+int32_t tSerializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  int64_t reserved64 = 0;
+  int8_t  msgVer = TSDB_SNAP_MSG_VER;
+
+  if (tStartEncode(&encoder) < 0) goto _err;
+  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
+  int16_t format = pOpts->format;
+  if (tEncodeI16(&encoder, format) < 0) goto _err;
+  if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+
+_err:
+  tEncoderClear(&encoder);
+  return -1;
+}
+
+int32_t tDeserializeTsdbRepOpts(void* buf, int32_t bufLen, STsdbRepOpts* pOpts) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  int64_t reserved64 = 0;
+  int8_t  msgVer = 0;
+
+  if (tStartDecode(&decoder) < 0) goto _err;
+  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
+  if (msgVer != TSDB_SNAP_MSG_VER) goto _err;
+  int16_t format = 0;
+  if (tDecodeI16(&decoder, &format) < 0) goto _err;
+  pOpts->format = format;
+  if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+
+_err:
+  tDecoderClear(&decoder);
+  return -1;
+}
+
+static int32_t tsdbRepOptsEstSize(STsdbRepOpts* pOpts) {
+  int32_t dataLen = 0;
+  dataLen += sizeof(SSyncTLV);
+  dataLen += tTsdbRepOptsDataLenCalc(pOpts);
+  return dataLen;
+}
+
+static int32_t tsdbRepOptsSerialize(STsdbRepOpts* pOpts, void* buf, int32_t bufLen) {
+  SSyncTLV* pSubHead = buf;
+  int32_t   offset = 0;
+  int32_t   tlen = 0;
+  if ((tlen = tSerializeTsdbRepOpts(pSubHead->val, bufLen, pOpts)) < 0) {
+    return -1;
+  }
+  pSubHead->typ = SNAP_DATA_RAW;
+  pSubHead->len = tlen;
+  offset += sizeof(*pSubHead) + tlen;
+  return offset;
+}
+
+// snap info
+static int32_t tsdbSnapPrepDealWithSnapInfo(SVnode* pVnode, SSnapshot* pSnap, STsdbRepOpts* pInfo) {
+  if (!pSnap->data) return 0;
+  int32_t code = -1;
+
+  SSyncTLV* pHead = (void*)pSnap->data;
+  int32_t   offset = 0;
+
+  while (offset + sizeof(*pHead) < pHead->len) {
+    SSyncTLV* pField = (void*)(pHead->val + offset);
+    offset += sizeof(*pField) + pField->len;
+    void*   buf = pField->val;
+    int32_t bufLen = pField->len;
+
+    switch (pField->typ) {
+      case SNAP_DATA_TSDB:
+      case SNAP_DATA_RSMA1:
+      case SNAP_DATA_RSMA2: {
+      } break;
+      case SNAP_DATA_RAW: {
+        if (tDeserializeTsdbRepOpts(buf, bufLen, pInfo) < 0) {
+          terrno = TSDB_CODE_INVALID_DATA_FMT;
+          tsdbError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+          goto _out;
+        }
+      } break;
+      default:
+        tsdbError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), pField->typ);
+        goto _out;
+    }
+  }
+
+  code = 0;
+_out:
+  return code;
+}
+
+int32_t tsdbSnapPrepDescription(SVnode* pVnode, SSnapshot* pSnap) {
+  ASSERT(pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
+  STsdbPartitionInfo  partitionInfo = {0};
+  int                 code = -1;
+  STsdbPartitionInfo* pInfo = &partitionInfo;
+
+  if (tsdbPartitionInfoInit(pVnode, pInfo) != 0) {
+    goto _out;
+  }
+
+  // deal with snap info for reply
+  STsdbRepOpts opts = {.format = TSDB_SNAP_REP_FMT_RAW};
+  if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    STsdbRepOpts leaderOpts = {0};
+    if (tsdbSnapPrepDealWithSnapInfo(pVnode, pSnap, &leaderOpts) < 0) {
+      tsdbError("vgId:%d, failed to deal with snap info for reply since %s", TD_VID(pVnode), terrstr());
+      goto _out;
+    }
+    opts.format = TMIN(opts.format, leaderOpts.format);
+  }
+
+  // info data realloc
+  const int32_t headLen = sizeof(SSyncTLV);
+  int32_t       bufLen = headLen;
+  bufLen += tsdbPartitionInfoEstSize(pInfo);
+  bufLen += tsdbRepOptsEstSize(&opts);
+  if (syncSnapInfoDataRealloc(pSnap, bufLen) != 0) {
+    tsdbError("vgId:%d, failed to realloc memory for data of snap info. bytes:%d", TD_VID(pVnode), bufLen);
+    goto _out;
+  }
+
+  // serialization
+  char*   buf = (void*)pSnap->data;
+  int32_t offset = headLen;
+  int32_t tlen = 0;
+
+  if ((tlen = tsdbPartitionInfoSerialize(pInfo, buf + offset, bufLen - offset)) < 0) {
+    tsdbError("vgId:%d, failed to serialize tsdb partition info since %s", TD_VID(pVnode), terrstr());
+    goto _out;
+  }
+  offset += tlen;
+  ASSERT(offset <= bufLen);
+
+  if ((tlen = tsdbRepOptsSerialize(&opts, buf + offset, bufLen - offset)) < 0) {
+    tsdbError("vgId:%d, failed to serialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+    goto _out;
+  }
+  offset += tlen;
+  ASSERT(offset <= bufLen);
+
+  // set header of info data
+  SSyncTLV* pHead = pSnap->data;
+  pHead->typ = pSnap->type;
+  pHead->len = offset - headLen;
+
+  tsdbInfo("vgId:%d, tsdb snap info prepared. type:%s, val length:%d", TD_VID(pVnode), TMSG_INFO(pHead->typ),
+           pHead->len);
+  code = 0;
+_out:
+  tsdbPartitionInfoClear(pInfo);
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -32,12 +32,12 @@ struct STsdbSnapReader {
   uint8_t* aBuf[5];
   SSkmInfo skmTb[1];
 
-  TSnapRangeArray* fsrArr;
+  TFileSetRangeArray* fsrArr;
 
   // context
   struct {
     int32_t      fsrArrIdx;
-    STSnapRange* fsr;
+    STFileSetRange* fsr;
     bool         isDataDone;
     bool         isTombDone;
   } ctx[1];
@@ -331,7 +331,7 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* reader, uint8_t** dat
 
     if (!(reader->blockData->nRow % 16)) {
       int64_t nData = tBlockDataSize(reader->blockData);
-      if (nData >= 1 * 1024 * 1024) {
+      if (nData >= TSDB_SNAP_DATA_PAYLOAD_SIZE) {
         break;
       }
     }
@@ -437,14 +437,14 @@ int32_t tsdbSnapReaderOpen(STsdb* tsdb, int64_t sver, int64_t ever, int8_t type,
   reader[0]->ever = ever;
   reader[0]->type = type;
 
-  code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TSnapRangeArray*)pRanges, &reader[0]->fsrArr);
+  code = tsdbFSCreateRefRangedSnapshot(tsdb->pFS, sver, ever, (TFileSetRangeArray*)pRanges, &reader[0]->fsrArr);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
   if (code) {
     tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode),
               __func__, lino, tstrerror(code), sver, ever, type);
-    tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
+    tsdbTFileSetRangeArrayDestroy(&reader[0]->fsrArr);
     taosMemoryFree(reader[0]);
     reader[0] = NULL;
   } else {
@@ -472,7 +472,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** reader) {
   TARRAY2_DESTROY(reader[0]->sttReaderArr, tsdbSttFileReaderClose);
   tsdbDataFileReaderClose(&reader[0]->dataReader);
 
-  tsdbSnapRangeArrayDestroy(&reader[0]->fsrArr);
+  tsdbFSDestroyRefRangedSnapshot(&reader[0]->fsrArr);
   tDestroyTSchema(reader[0]->skmTb->pTSchema);
 
   for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->aBuf); ++i) {
@@ -1061,7 +1061,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang
   writer[0]->compactVersion = INT64_MAX;
   writer[0]->now = taosGetTimestampMs();
 
-  code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TSnapRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr);
+  code = tsdbFSCreateCopyRangedSnapshot(pTsdb->pFS, (TFileSetRangeArray*)pRanges, &writer[0]->fsetArr, writer[0]->fopArr);
   TSDB_CHECK_CODE(code, lino, _exit);
 
 _exit:
@@ -1125,7 +1125,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) {
   tsdbDataFileReaderClose(&writer[0]->ctx->dataReader);
 
   TARRAY2_DESTROY(writer[0]->fopArr, NULL);
-  tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr);
+  tsdbFSDestroyCopyRangedSnapshot(&writer[0]->fsetArr);
 
   for (int32_t i = 0; i < ARRAY_SIZE(writer[0]->aBuf); ++i) {
     tFree(writer[0]->aBuf[i]);
@@ -1167,439 +1167,3 @@ _exit:
   }
   return code;
 }
-
-// snap part
-static int32_t tsdbSnapPartCmprFn(STsdbSnapPartition* x, STsdbSnapPartition* y) {
-  if (x->fid < y->fid) return -1;
-  if (x->fid > y->fid) return 1;
-  return 0;
-}
-
-static int32_t tVersionRangeCmprFn(SVersionRange* x, SVersionRange* y) {
-  if (x->minVer < y->minVer) return -1;
-  if (x->minVer > y->minVer) return 1;
-  if (x->maxVer < y->maxVer) return -1;
-  if (x->maxVer > y->maxVer) return 1;
-  return 0;
-}
-
-static int32_t tsdbSnapRangeCmprFn(STSnapRange* x, STSnapRange* y) {
-  if (x->fid < y->fid) return -1;
-  if (x->fid > y->fid) return 1;
-  return 0;
-}
-
-STsdbSnapPartition* tsdbSnapPartitionCreate() {
-  STsdbSnapPartition* pSP = taosMemoryCalloc(1, sizeof(STsdbSnapPartition));
-  if (pSP == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
-    TARRAY2_INIT(&pSP->verRanges[i]);
-  }
-  return pSP;
-}
-
-void tsdbSnapPartitionClear(STsdbSnapPartition** ppSP) {
-  if (ppSP == NULL || ppSP[0] == NULL) {
-    return;
-  }
-  for (int32_t i = 0; i < TSDB_SNAP_RANGE_TYP_MAX; i++) {
-    TARRAY2_DESTROY(&ppSP[0]->verRanges[i], NULL);
-  }
-  taosMemoryFree(ppSP[0]);
-  ppSP[0] = NULL;
-}
-
-static int32_t tsdbFTypeToSRangeTyp(tsdb_ftype_t ftype) {
-  switch (ftype) {
-    case TSDB_FTYPE_HEAD:
-      return TSDB_SNAP_RANGE_TYP_HEAD;
-    case TSDB_FTYPE_DATA:
-      return TSDB_SNAP_RANGE_TYP_DATA;
-    case TSDB_FTYPE_SMA:
-      return TSDB_SNAP_RANGE_TYP_SMA;
-    case TSDB_FTYPE_TOMB:
-      return TSDB_SNAP_RANGE_TYP_TOMB;
-    case TSDB_FTYPE_STT:
-      return TSDB_SNAP_RANGE_TYP_STT;
-  }
-  return TSDB_SNAP_RANGE_TYP_MAX;
-}
-
-static int32_t tsdbTFileSetToSnapPart(STFileSet* fset, STsdbSnapPartition** ppSP) {
-  STsdbSnapPartition* p = tsdbSnapPartitionCreate();
-  if (p == NULL) {
-    goto _err;
-  }
-
-  p->fid = fset->fid;
-
-  int32_t code = 0;
-  int32_t typ = 0;
-  int32_t corrupt = false;
-  int32_t count = 0;
-  for (int32_t ftype = TSDB_FTYPE_MIN; ftype < TSDB_FTYPE_MAX; ++ftype) {
-    if (fset->farr[ftype] == NULL) continue;
-    typ = tsdbFTypeToSRangeTyp(ftype);
-    ASSERT(typ < TSDB_SNAP_RANGE_TYP_MAX);
-    STFile* f = fset->farr[ftype]->f;
-    if (f->maxVer > fset->maxVerValid) {
-      corrupt = true;
-      tsdbError("skip incomplete data file: fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
-                ", ftype: %d",
-                fset->fid, fset->maxVerValid, f->minVer, f->maxVer, ftype);
-      continue;
-    }
-    count++;
-    SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
-    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-    ASSERT(code == 0);
-  }
-
-  typ = TSDB_SNAP_RANGE_TYP_STT;
-  const SSttLvl* lvl;
-  TARRAY2_FOREACH(fset->lvlArr, lvl) {
-    STFileObj* fobj;
-    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
-      STFile* f = fobj->f;
-      if (f->maxVer > fset->maxVerValid) {
-        corrupt = true;
-        tsdbError("skip incomplete stt file.fid:%d, maxVerValid:%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64
-                  ", ftype: %d",
-                  fset->fid, fset->maxVerValid, f->minVer, f->maxVer, typ);
-        continue;
-      }
-      count++;
-      SVersionRange vr = {.minVer = f->minVer, .maxVer = f->maxVer};
-      code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-      ASSERT(code == 0);
-    }
-  }
-  if (corrupt && count == 0) {
-    SVersionRange vr = {.minVer = VERSION_MIN, .maxVer = fset->maxVerValid};
-    code = TARRAY2_SORT_INSERT(&p->verRanges[typ], vr, tVersionRangeCmprFn);
-    ASSERT(code == 0);
-  }
-  ppSP[0] = p;
-  return 0;
-
-_err:
-  tsdbSnapPartitionClear(&p);
-  return -1;
-}
-
-STsdbSnapPartList* tsdbSnapPartListCreate() {
-  STsdbSnapPartList* pList = taosMemoryCalloc(1, sizeof(STsdbSnapPartList));
-  if (pList == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
-  TARRAY2_INIT(pList);
-  return pList;
-}
-
-static STsdbSnapPartList* tsdbGetSnapPartList(STFileSystem* fs) {
-  STsdbSnapPartList* pList = tsdbSnapPartListCreate();
-  if (pList == NULL) {
-    return NULL;
-  }
-
-  int32_t code = 0;
-  taosThreadMutexLock(&fs->tsdb->mutex);
-  STFileSet* fset;
-  TARRAY2_FOREACH(fs->fSetArr, fset) {
-    STsdbSnapPartition* pItem = NULL;
-    if (tsdbTFileSetToSnapPart(fset, &pItem) < 0) {
-      code = -1;
-      break;
-    }
-    ASSERT(pItem != NULL);
-    code = TARRAY2_SORT_INSERT(pList, pItem, tsdbSnapPartCmprFn);
-    ASSERT(code == 0);
-  }
-  taosThreadMutexUnlock(&fs->tsdb->mutex);
-
-  if (code) {
-    TARRAY2_DESTROY(pList, tsdbSnapPartitionClear);
-    taosMemoryFree(pList);
-    pList = NULL;
-  }
-  return pList;
-}
-
-int32_t tTsdbSnapPartListDataLenCalc(STsdbSnapPartList* pList) {
-  int32_t hdrLen = sizeof(int32_t);
-  int32_t datLen = 0;
-
-  int8_t  msgVer = 1;
-  int32_t len = TARRAY2_SIZE(pList);
-  hdrLen += sizeof(msgVer);
-  hdrLen += sizeof(len);
-  datLen += hdrLen;
-
-  for (int32_t u = 0; u < len; u++) {
-    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
-    int32_t             typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    int32_t             uItem = 0;
-    uItem += sizeof(STsdbSnapPartition);
-    uItem += sizeof(typMax);
-
-    for (int32_t i = 0; i < typMax; i++) {
-      int32_t iLen = TARRAY2_SIZE(&p->verRanges[i]);
-      int32_t jItem = 0;
-      jItem += sizeof(SVersionRange);
-      jItem += sizeof(int64_t);
-      uItem += sizeof(iLen) + jItem * iLen;
-    }
-    datLen += uItem;
-  }
-  return datLen;
-}
-
-int32_t tSerializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
-  SEncoder encoder = {0};
-  tEncoderInit(&encoder, buf, bufLen);
-
-  int8_t  reserved8 = 0;
-  int16_t reserved16 = 0;
-  int64_t reserved64 = 0;
-
-  int8_t  msgVer = 1;
-  int32_t len = TARRAY2_SIZE(pList);
-
-  if (tStartEncode(&encoder) < 0) goto _err;
-  if (tEncodeI8(&encoder, msgVer) < 0) goto _err;
-  if (tEncodeI32(&encoder, len) < 0) goto _err;
-
-  for (int32_t u = 0; u < len; u++) {
-    STsdbSnapPartition* p = TARRAY2_GET(pList, u);
-    if (tEncodeI64(&encoder, p->fid) < 0) goto _err;
-    if (tEncodeI8(&encoder, p->stat) < 0) goto _err;
-    if (tEncodeI8(&encoder, reserved8) < 0) goto _err;
-    if (tEncodeI16(&encoder, reserved16) < 0) goto _err;
-
-    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    if (tEncodeI32(&encoder, typMax) < 0) goto _err;
-
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &p->verRanges[i];
-      int32_t        iLen = TARRAY2_SIZE(iList);
-
-      if (tEncodeI32(&encoder, iLen) < 0) goto _err;
-      for (int32_t j = 0; j < iLen; j++) {
-        SVersionRange r = TARRAY2_GET(iList, j);
-        if (tEncodeI64(&encoder, r.minVer) < 0) goto _err;
-        if (tEncodeI64(&encoder, r.maxVer) < 0) goto _err;
-        if (tEncodeI64(&encoder, reserved64) < 0) goto _err;
-      }
-    }
-  }
-
-  tEndEncode(&encoder);
-  int32_t tlen = encoder.pos;
-  tEncoderClear(&encoder);
-  return tlen;
-
-_err:
-  tEncoderClear(&encoder);
-  return -1;
-}
-
-int32_t tDeserializeTsdbSnapPartList(void* buf, int32_t bufLen, STsdbSnapPartList* pList) {
-  SDecoder decoder = {0};
-  tDecoderInit(&decoder, buf, bufLen);
-
-  int8_t  reserved8 = 0;
-  int16_t reserved16 = 0;
-  int64_t reserved64 = 0;
-
-  STsdbSnapPartition* p = NULL;
-
-  int8_t  msgVer = 0;
-  int32_t len = 0;
-  if (tStartDecode(&decoder) < 0) goto _err;
-  if (tDecodeI8(&decoder, &msgVer) < 0) goto _err;
-  if (tDecodeI32(&decoder, &len) < 0) goto _err;
-
-  for (int32_t u = 0; u < len; u++) {
-    p = tsdbSnapPartitionCreate();
-    if (p == NULL) goto _err;
-    if (tDecodeI64(&decoder, &p->fid) < 0) goto _err;
-    if (tDecodeI8(&decoder, &p->stat) < 0) goto _err;
-    if (tDecodeI8(&decoder, &reserved8) < 0) goto _err;
-    if (tDecodeI16(&decoder, &reserved16) < 0) goto _err;
-
-    int32_t typMax = 0;
-    if (tDecodeI32(&decoder, &typMax) < 0) goto _err;
-
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &p->verRanges[i];
-      int32_t        iLen = 0;
-      if (tDecodeI32(&decoder, &iLen) < 0) goto _err;
-      for (int32_t j = 0; j < iLen; j++) {
-        SVersionRange r = {0};
-        if (tDecodeI64(&decoder, &r.minVer) < 0) goto _err;
-        if (tDecodeI64(&decoder, &r.maxVer) < 0) goto _err;
-        if (tDecodeI64(&decoder, &reserved64) < 0) goto _err;
-        TARRAY2_APPEND(iList, r);
-      }
-    }
-    TARRAY2_APPEND(pList, p);
-    p = NULL;
-  }
-
-  tEndDecode(&decoder);
-  tDecoderClear(&decoder);
-  return 0;
-
-_err:
-  if (p) {
-    tsdbSnapPartitionClear(&p);
-  }
-  tDecoderClear(&decoder);
-  return -1;
-}
-
-int32_t tsdbSnapPartListToRangeDiff(STsdbSnapPartList* pList, TSnapRangeArray** ppRanges) {
-  TSnapRangeArray* pDiff = taosMemoryCalloc(1, sizeof(TSnapRangeArray));
-  if (pDiff == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  TARRAY2_INIT(pDiff);
-
-  STsdbSnapPartition* part;
-  TARRAY2_FOREACH(pList, part) {
-    STSnapRange* r = taosMemoryCalloc(1, sizeof(STSnapRange));
-    if (r == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
-    int64_t maxVerValid = -1;
-    int32_t typMax = TSDB_SNAP_RANGE_TYP_MAX;
-    for (int32_t i = 0; i < typMax; i++) {
-      SVerRangeList* iList = &part->verRanges[i];
-      SVersionRange  vr = {0};
-      TARRAY2_FOREACH(iList, vr) {
-        if (vr.maxVer < vr.minVer) {
-          continue;
-        }
-        maxVerValid = TMAX(maxVerValid, vr.maxVer);
-      }
-    }
-    r->fid = part->fid;
-    r->sver = maxVerValid + 1;
-    r->ever = VERSION_MAX;
-    tsdbDebug("range diff fid:%" PRId64 ", sver:%" PRId64 ", ever:%" PRId64, part->fid, r->sver, r->ever);
-    int32_t code = TARRAY2_SORT_INSERT(pDiff, r, tsdbSnapRangeCmprFn);
-    ASSERT(code == 0);
-  }
-  ppRanges[0] = pDiff;
-
-  tsdbInfo("pDiff size:%d", TARRAY2_SIZE(pDiff));
-  return 0;
-
-_err:
-  if (pDiff) {
-    tsdbSnapRangeArrayDestroy(&pDiff);
-  }
-  return -1;
-}
-
-void tsdbSnapRangeArrayDestroy(TSnapRangeArray** ppSnap) {
-  if (ppSnap && ppSnap[0]) {
-    TARRAY2_DESTROY(ppSnap[0], tsdbTSnapRangeClear);
-    taosMemoryFree(ppSnap[0]);
-    ppSnap[0] = NULL;
-  }
-}
-
-void tsdbSnapPartListDestroy(STsdbSnapPartList** ppList) {
-  if (ppList == NULL || ppList[0] == NULL) return;
-
-  TARRAY2_DESTROY(ppList[0], tsdbSnapPartitionClear);
-  taosMemoryFree(ppList[0]);
-  ppList[0] = NULL;
-}
-
-ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
-  if (!VND_IS_RSMA(pVnode)) {
-    return pVnode->pTsdb->pFS->fsstate;
-  }
-  for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
-    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
-    if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
-      return TSDB_FS_STATE_INCOMPLETE;
-    }
-  }
-  return TSDB_FS_STATE_NORMAL;
-}
-
-int32_t tsdbSnapGetDetails(SVnode* pVnode, SSnapshot* pSnap) {
-  int                code = -1;
-  int32_t            tsdbMaxCnt = (!VND_IS_RSMA(pVnode) ? 1 : TSDB_RETENTION_MAX);
-  int32_t            subTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
-  STsdbSnapPartList* pLists[TSDB_RETENTION_MAX] = {0};
-
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, j);
-    pLists[j] = tsdbGetSnapPartList(pTsdb->pFS);
-    if (pLists[j] == NULL) goto _out;
-  }
-
-  // estimate bufLen and prepare
-  int32_t bufLen = sizeof(SSyncTLV);  // typ: TDMT_SYNC_PREP_SNAPSHOT or TDMT_SYNC_PREP_SNAPSOT_REPLY
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    bufLen += sizeof(SSyncTLV);  // subTyps[j]
-    bufLen += tTsdbSnapPartListDataLenCalc(pLists[j]);
-  }
-
-  tsdbInfo("vgId:%d, allocate %d bytes for data of snapshot info.", TD_VID(pVnode), bufLen);
-
-  void* data = taosMemoryRealloc(pSnap->data, bufLen);
-  if (data == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    tsdbError("vgId:%d, failed to realloc memory for data of snapshot info. bytes:%d", TD_VID(pVnode), bufLen);
-    goto _out;
-  }
-  pSnap->data = data;
-
-  // header
-  SSyncTLV* head = data;
-  head->len = 0;
-  head->typ = pSnap->type;
-  int32_t offset = sizeof(SSyncTLV);
-  int32_t tlen = 0;
-
-  // fill snapshot info
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    }
-
-    //  subHead
-    SSyncTLV* subHead = (void*)((char*)data + offset);
-    subHead->typ = subTyps[j];
-    ASSERT(subHead->val == (char*)data + offset + sizeof(SSyncTLV));
-
-    if ((tlen = tSerializeTsdbSnapPartList(subHead->val, bufLen - offset - sizeof(SSyncTLV), pLists[j])) < 0) {
-      tsdbError("vgId:%d, failed to serialize snap partition list of tsdb %d since %s", TD_VID(pVnode), j, terrstr());
-      goto _out;
-    }
-    subHead->len = tlen;
-    offset += sizeof(SSyncTLV) + tlen;
-  }
-
-  head->len = offset - sizeof(SSyncTLV);
-  ASSERT(offset <= bufLen);
-  code = 0;
-
-_out:
-  for (int32_t j = 0; j < tsdbMaxCnt; ++j) {
-    if (pLists[j] == NULL) continue;
-    tsdbSnapPartListDestroy(&pLists[j]);
-  }
-
-  return code;
-}

source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c

@@ -0,0 +1,599 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdb.h"
+#include "tsdbDataFileRAW.h"
+#include "tsdbFS2.h"
+#include "tsdbFSetRAW.h"
+
+static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader);
+
+// reader
+typedef struct SDataFileRAWReaderIter {
+  int32_t count;
+  int32_t idx;
+} SDataFileRAWReaderIter;
+
+typedef struct STsdbSnapRAWReader {
+  STsdb*  tsdb;
+  int64_t ever;
+  int8_t  type;
+
+  TFileSetArray* fsetArr;
+
+  // context
+  struct {
+    int32_t    fsetArrIdx;
+    STFileSet* fset;
+    bool       isDataDone;
+  } ctx[1];
+
+  // reader
+  SDataFileRAWReaderArray dataReaderArr[1];
+
+  // iter
+  SDataFileRAWReaderIter dataIter[1];
+} STsdbSnapRAWReader;
+
+int32_t tsdbSnapRAWReaderOpen(STsdb* tsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader[0] = taosMemoryCalloc(1, sizeof(STsdbSnapRAWReader));
+  if (reader[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  reader[0]->tsdb = tsdb;
+  reader[0]->ever = ever;
+  reader[0]->type = type;
+
+  code = tsdbFSCreateRefSnapshot(tsdb->pFS, &reader[0]->fsetArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s, sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), __func__,
+              lino, tstrerror(code), ever, type);
+    tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+    taosMemoryFree(reader[0]);
+    reader[0] = NULL;
+  } else {
+    tsdbInfo("vgId:%d tsdb snapshot reader opened. sver:0, ever:%" PRId64 " type:%d", TD_VID(tsdb->pVnode), ever, type);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** reader) {
+  if (reader[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb* tsdb = reader[0]->tsdb;
+
+  TARRAY2_DESTROY(reader[0]->dataReaderArr, tsdbDataFileRAWReaderClose);
+  tsdbFSDestroyRefSnapshot(&reader[0]->fsetArr);
+  taosMemoryFree(reader[0]);
+  reader[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetOpenReader(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // data
+  for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ftype++) {
+    if (reader->ctx->fset->farr[ftype] == NULL) {
+      continue;
+    }
+    STFileObj*               fobj = reader->ctx->fset->farr[ftype];
+    SDataFileRAWReader*      dataReader;
+    SDataFileRAWReaderConfig config = {
+        .tsdb = reader->tsdb,
+        .szPage = reader->tsdb->pVnode->config.tsdbPageSize,
+        .file = fobj->f[0],
+    };
+    code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = TARRAY2_APPEND(reader->dataReaderArr, dataReader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  // stt
+  SSttLvl* lvl;
+  TARRAY2_FOREACH(reader->ctx->fset->lvlArr, lvl) {
+    STFileObj* fobj;
+    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
+      SDataFileRAWReader*      dataReader;
+      SDataFileRAWReaderConfig config = {
+          .tsdb = reader->tsdb,
+          .szPage = reader->tsdb->pVnode->config.tsdbPageSize,
+          .file = fobj->f[0],
+      };
+      code = tsdbDataFileRAWReaderOpen(NULL, &config, &dataReader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      code = TARRAY2_APPEND(reader->dataReaderArr, dataReader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetCloseReader(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  TARRAY2_CLEAR(reader->dataReaderArr, tsdbDataFileRAWReaderClose);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetOpenIter(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader->dataIter->count = TARRAY2_SIZE(reader->dataReaderArr);
+  reader->dataIter->idx = 0;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadFileSetCloseIter(STsdbSnapRAWReader* reader) {
+  reader->dataIter->count = 0;
+  reader->dataIter->idx = 0;
+  return 0;
+}
+
+static int64_t tsdbSnapRAWReadPeek(SDataFileRAWReader* reader) {
+  int64_t size = TMIN(reader->config->file.size - reader->ctx->offset, TSDB_SNAP_DATA_PAYLOAD_SIZE);
+  return size;
+}
+
+static SDataFileRAWReader* tsdbSnapRAWReaderIterNext(STsdbSnapRAWReader* reader) {
+  ASSERT(reader->dataIter->idx <= reader->dataIter->count);
+
+  while (reader->dataIter->idx < reader->dataIter->count) {
+    SDataFileRAWReader* dataReader = TARRAY2_GET(reader->dataReaderArr, reader->dataIter->idx);
+    ASSERT(dataReader);
+    if (dataReader->ctx->offset < dataReader->config->file.size) {
+      return dataReader;
+    }
+    reader->dataIter->idx++;
+  }
+  return NULL;
+}
+
+static int32_t tsdbSnapRAWReadNext(STsdbSnapRAWReader* reader, SSnapDataHdr** ppData) {
+  int32_t code = 0;
+  int32_t lino = 0;
+  int8_t  type = reader->type;
+  ppData[0] = NULL;
+
+  SDataFileRAWReader* dataReader = tsdbSnapRAWReaderIterNext(reader);
+  if (dataReader == NULL) {
+    return 0;
+  }
+
+  // prepare
+  int64_t dataLength = tsdbSnapRAWReadPeek(dataReader);
+  ASSERT(dataLength > 0);
+
+  void* pBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + sizeof(STsdbDataRAWBlockHeader) + dataLength);
+  if (pBuf == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  SSnapDataHdr* pHdr = pBuf;
+  pHdr->type = type;
+  pHdr->size = sizeof(STsdbDataRAWBlockHeader) + dataLength;
+
+  // read
+  STsdbDataRAWBlockHeader* pBlock = (void*)pHdr->data;
+  pBlock->offset = dataReader->ctx->offset;
+  pBlock->dataLength = dataLength;
+
+  code = tsdbDataFileRAWReadBlockData(dataReader, pBlock);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  // finish
+  dataReader->ctx->offset += pBlock->dataLength;
+  ASSERT(dataReader->ctx->offset <= dataReader->config->file.size);
+  ppData[0] = pBuf;
+
+_exit:
+  if (code) {
+    taosMemoryFree(pBuf);
+    pBuf = NULL;
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadData(STsdbSnapRAWReader* reader, uint8_t** ppData) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWReadNext(reader, (SSnapDataHdr**)ppData);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadBegin(STsdbSnapRAWReader* reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(reader->ctx->fset == NULL);
+
+  if (reader->ctx->fsetArrIdx < TARRAY2_SIZE(reader->fsetArr)) {
+    reader->ctx->fset = TARRAY2_GET(reader->fsetArr, reader->ctx->fsetArrIdx++);
+    reader->ctx->isDataDone = false;
+
+    code = tsdbSnapRAWReadFileSetOpenReader(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbSnapRAWReadFileSetOpenIter(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWReadEnd(STsdbSnapRAWReader* reader) {
+  tsdbSnapRAWReadFileSetCloseIter(reader);
+  tsdbSnapRAWReadFileSetCloseReader(reader);
+  reader->ctx->fset = NULL;
+  return 0;
+}
+
+int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* reader, uint8_t** data) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  data[0] = NULL;
+
+  for (;;) {
+    if (reader->ctx->fset == NULL) {
+      code = tsdbSnapRAWReadBegin(reader);
+      TSDB_CHECK_CODE(code, lino, _exit);
+
+      if (reader->ctx->fset == NULL) {
+        break;
+      }
+    }
+
+    if (!reader->ctx->isDataDone) {
+      code = tsdbSnapRAWReadData(reader, data);
+      TSDB_CHECK_CODE(code, lino, _exit);
+      if (data[0]) {
+        goto _exit;
+      } else {
+        reader->ctx->isDataDone = true;
+      }
+    }
+
+    code = tsdbSnapRAWReadEnd(reader);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->tsdb->pVnode), code, lino);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(reader->tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+// writer
+struct STsdbSnapRAWWriter {
+  STsdb*  tsdb;
+  int64_t sver;
+  int64_t ever;
+  int32_t minutes;
+  int8_t  precision;
+  int32_t minRow;
+  int32_t maxRow;
+  int8_t  cmprAlg;
+  int64_t commitID;
+  int32_t szPage;
+  int64_t compactVersion;
+  int64_t now;
+
+  TFileSetArray* fsetArr;
+  TFileOpArray   fopArr[1];
+
+  struct {
+    bool       fsetWriteBegin;
+    int32_t    fid;
+    STFileSet* fset;
+    SDiskID    did;
+    int64_t    cid;
+    int64_t    level;
+
+    // writer
+    SFSetRAWWriter* fsetWriter;
+  } ctx[1];
+};
+
+int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // start to write
+  writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
+  if (writer[0] == NULL) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer[0]->tsdb = pTsdb;
+  writer[0]->ever = ever;
+  writer[0]->minutes = pTsdb->keepCfg.days;
+  writer[0]->precision = pTsdb->keepCfg.precision;
+  writer[0]->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
+  writer[0]->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
+  writer[0]->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
+  writer[0]->commitID = tsdbFSAllocEid(pTsdb->pFS);
+  writer[0]->szPage = pTsdb->pVnode->config.tsdbPageSize;
+  writer[0]->compactVersion = INT64_MAX;
+  writer[0]->now = taosGetTimestampMs();
+
+  code = tsdbFSCreateCopySnapshot(pTsdb->pFS, &writer[0]->fsetArr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+  } else {
+    tsdbInfo("vgId:%d %s done, sver:0, ever:%" PRId64, TD_VID(pTsdb->pVnode), __func__, ever);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetOpenIter(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetCloseIter(STsdbSnapRAWWriter* writer) { return 0; }
+
+static int32_t tsdbSnapRAWWriteFileSetOpenWriter(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SFSetRAWWriterConfig config = {
+      .tsdb = writer->tsdb,
+      .szPage = writer->szPage,
+      .fid = writer->ctx->fid,
+      .cid = writer->commitID,
+      .did = writer->ctx->did,
+      .level = writer->ctx->level,
+  };
+
+  code = tsdbFSetRAWWriterOpen(&config, &writer->ctx->fsetWriter);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetCloseWriter(STsdbSnapRAWWriter* writer) {
+  return tsdbFSetRAWWriterClose(&writer->ctx->fsetWriter, 0, writer->fopArr);
+}
+
+static int32_t tsdbSnapRAWWriteFileSetBegin(STsdbSnapRAWWriter* writer, int32_t fid) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  ASSERT(writer->ctx->fsetWriteBegin == false);
+
+  STFileSet* fset = &(STFileSet){.fid = fid};
+
+  writer->ctx->fid = fid;
+  STFileSet** fsetPtr = TARRAY2_SEARCH(writer->fsetArr, &fset, tsdbTFileSetCmprFn, TD_EQ);
+  writer->ctx->fset = (fsetPtr == NULL) ? NULL : *fsetPtr;
+
+  int32_t level = tsdbFidLevel(fid, &writer->tsdb->keepCfg, taosGetTimestampSec());
+  if (tfsAllocDisk(writer->tsdb->pVnode->pTfs, level, &writer->ctx->did)) {
+    code = TSDB_CODE_NO_AVAIL_DISK;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  tfsMkdirRecurAt(writer->tsdb->pVnode->pTfs, writer->tsdb->path, writer->ctx->did);
+
+  code = tsdbSnapRAWWriteFileSetOpenWriter(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->level = level;
+  writer->ctx->fsetWriteBegin = true;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteFileSetEnd(STsdbSnapRAWWriter* writer) {
+  if (!writer->ctx->fsetWriteBegin) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  // close write
+  code = tsdbSnapRAWWriteFileSetCloseWriter(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->fsetWriteBegin = false;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWWriteFileSetEnd(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  code = tsdbFSEditBegin(writer->tsdb->pFS, writer->fopArr, TSDB_FEDIT_COMMIT);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  } else {
+    tsdbDebug("vgId:%d %s done", TD_VID(writer->tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdb* tsdb = writer[0]->tsdb;
+
+  if (rollback) {
+    code = tsdbFSEditAbort(writer[0]->tsdb->pFS);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  } else {
+    taosThreadMutexLock(&writer[0]->tsdb->mutex);
+
+    code = tsdbFSEditCommit(writer[0]->tsdb->pFS);
+    if (code) {
+      taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+
+    writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;
+
+    taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
+  }
+
+  TARRAY2_DESTROY(writer[0]->fopArr, NULL);
+  tsdbFSDestroyCopySnapshot(&writer[0]->fsetArr);
+
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
+  } else {
+    tsdbInfo("vgId:%d %s done", TD_VID(tsdb->pVnode), __func__);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteTimeSeriesData(STsdbSnapRAWWriter* writer, STsdbDataRAWBlockHeader* bHdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbFSetRAWWriteBlockData(writer->ctx->fsetWriter, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbSnapRAWWriteData(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  STsdbDataRAWBlockHeader* bHdr = (void*)hdr->data;
+  int32_t                  fid = bHdr->file.fid;
+  if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) {
+    code = tsdbSnapRAWWriteFileSetEnd(writer);
+    TSDB_CHECK_CODE(code, lino, _exit);
+
+    code = tsdbSnapRAWWriteFileSetBegin(writer, fid);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  code = tsdbSnapRAWWriteTimeSeriesData(writer, bHdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* writer, SSnapDataHdr* hdr) {
+  ASSERT(hdr->type == SNAP_DATA_RAW);
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbSnapRAWWriteData(writer, hdr);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    tsdbError("vgId:%d %s failed at line %d since %s, type:%d index:%" PRId64 " size:%" PRId64,
+              TD_VID(writer->tsdb->pVnode), __func__, lino, tstrerror(code), hdr->type, hdr->index, hdr->size);
+  } else {
+    tsdbDebug("vgId:%d %s done, type:%d index:%" PRId64 " size:%" PRId64, TD_VID(writer->tsdb->pVnode), __func__,
+              hdr->type, hdr->index, hdr->size);
+  }
+  return code;
+}

source/dnode/vnode/src/vnd/vnodeAsync.c

@@ -177,12 +177,15 @@ static int32_t vnodeAsyncTaskDone(SVAsync *async, SVATask *task) {
 }
 
 static int32_t vnodeAsyncCancelAllTasks(SVAsync *async) {
-  for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
-    while (async->queue[i].next != &async->queue[i]) {
-      SVATask *task = async->queue[i].next;
-      task->prev->next = task->next;
-      task->next->prev = task->prev;
-      vnodeAsyncTaskDone(async, task);
+  while (async->queue[0].next != &async->queue[0] || async->queue[1].next != &async->queue[1] ||
+         async->queue[2].next != &async->queue[2]) {
+    for (int32_t i = 0; i < EVA_PRIORITY_MAX; i++) {
+      while (async->queue[i].next != &async->queue[i]) {
+        SVATask *task = async->queue[i].next;
+        task->prev->next = task->next;
+        task->next->prev = task->prev;
+        vnodeAsyncTaskDone(async, task);
+      }
     }
   }
   return 0;

source/dnode/vnode/src/vnd/vnodeSnapshot.c

@@ -16,6 +16,26 @@
 #include "tsdb.h"
 #include "vnd.h"
 
+static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TFileSetRangeArray **ppRanges) {
+  int32_t            code = -1;
+  STsdbFSetPartList *pList = tsdbFSetPartListCreate();
+  if (pList == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _out;
+  }
+  if (tDeserializeTsdbFSetPartList(buf, bufLen, pList) < 0) {
+    terrno = TSDB_CODE_INVALID_DATA_FMT;
+    goto _out;
+  }
+  if (tsdbFSetPartListToRangeDiff(pList, ppRanges) < 0) {
+    goto _out;
+  }
+  code = 0;
+_out:
+  tsdbFSetPartListDestroy(&pList);
+  return code;
+}
+
 // SVSnapReader ========================================================
 struct SVSnapReader {
   SVnode *pVnode;
@@ -29,8 +49,12 @@ struct SVSnapReader {
   SMetaSnapReader *pMetaReader;
   // tsdb
   int8_t           tsdbDone;
-  TSnapRangeArray *pRanges;
+  TFileSetRangeArray *pRanges;
   STsdbSnapReader *pTsdbReader;
+  // tsdb raw
+  int8_t              tsdbRAWDone;
+  STsdbSnapRAWReader *pTsdbRAWReader;
+
   // tq
   int8_t              tqHandleDone;
   STqSnapReader      *pTqSnapReader;
@@ -45,31 +69,11 @@ struct SVSnapReader {
   SStreamStateReader *pStreamStateReader;
   // rsma
   int8_t           rsmaDone;
-  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
+  TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapReader *pRsmaReader;
 };
 
-static int32_t vnodeExtractSnapInfoDiff(void *buf, int32_t bufLen, TSnapRangeArray **ppRanges) {
-  int32_t            code = -1;
-  STsdbSnapPartList *pList = tsdbSnapPartListCreate();
-  if (pList == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    goto _out;
-  }
-  if (tDeserializeTsdbSnapPartList(buf, bufLen, pList) < 0) {
-    terrno = TSDB_CODE_INVALID_DATA_FMT;
-    goto _out;
-  }
-  if (tsdbSnapPartListToRangeDiff(pList, ppRanges) < 0) {
-    goto _out;
-  }
-  code = 0;
-_out:
-  tsdbSnapPartListDestroy(&pList);
-  return code;
-}
-
-static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
+static TFileSetRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int32_t tsdbTyp) {
   ASSERTS(sizeof(pReader->pRsmaRanges) / sizeof(pReader->pRsmaRanges[0]) == 2, "Unexpected array size");
   switch (tsdbTyp) {
     case SNAP_DATA_TSDB:
@@ -83,37 +87,66 @@ static TSnapRangeArray **vnodeSnapReaderGetTsdbRanges(SVSnapReader *pReader, int
   }
 }
 
-static int32_t vnodeSnapReaderDoSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
+static int32_t vnodeSnapReaderDealWithSnapInfo(SVSnapReader *pReader, SSnapshotParam *pParam) {
   SVnode *pVnode = pReader->pVnode;
   int32_t code = -1;
 
   if (pParam->data) {
+    // decode
     SSyncTLV *datHead = (void *)pParam->data;
     if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
       terrno = TSDB_CODE_INVALID_DATA_FMT;
       goto _out;
     }
 
-    TSnapRangeArray **ppRanges = NULL;
-    int32_t           offset = 0;
+    STsdbRepOpts         tsdbOpts = {0};
+    TFileSetRangeArray **ppRanges = NULL;
+    int32_t              offset = 0;
 
     while (offset + sizeof(SSyncTLV) < datHead->len) {
       SSyncTLV *subField = (void *)(datHead->val + offset);
       offset += sizeof(SSyncTLV) + subField->len;
       void   *buf = subField->val;
       int32_t bufLen = subField->len;
-      ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
-      if (ppRanges == NULL) {
-        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
-        goto _out;
-      }
-      if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
-        vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
-        goto _out;
+
+      switch (subField->typ) {
+        case SNAP_DATA_TSDB:
+        case SNAP_DATA_RSMA1:
+        case SNAP_DATA_RSMA2: {
+          ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, subField->typ);
+          if (ppRanges == NULL) {
+            vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+            goto _out;
+          }
+          if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
+            vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
+            goto _out;
+          }
+        } break;
+        case SNAP_DATA_RAW: {
+          if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
+            vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+            goto _out;
+          }
+        } break;
+        default:
+          vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
+          goto _out;
       }
     }
-  }
 
+    // toggle snap replication mode
+    vInfo("vgId:%d, vnode snap reader supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
+    if (pReader->sver == 0 && tsdbOpts.format == TSDB_SNAP_REP_FMT_RAW) {
+      pReader->tsdbDone = true;
+    } else {
+      pReader->tsdbRAWDone = true;
+    }
+
+    ASSERT(pReader->tsdbDone != pReader->tsdbRAWDone);
+    vInfo("vgId:%d, vnode snap writer enabled replication mode: %s", TD_VID(pVnode),
+          (pReader->tsdbDone ? "raw" : "normal"));
+  }
   code = 0;
 _out:
   return code;
@@ -135,7 +168,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapReader
   pReader->ever = ever;
 
   // snapshot info
-  if (vnodeSnapReaderDoSnapInfo(pReader, pParam) < 0) {
+  if (vnodeSnapReaderDealWithSnapInfo(pReader, pParam) < 0) {
     goto _err;
   }
 
@@ -152,9 +185,9 @@ _err:
 static void vnodeSnapReaderDestroyTsdbRanges(SVSnapReader *pReader) {
   int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
   for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
-    TSnapRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]);
+    TFileSetRangeArray **ppRanges = vnodeSnapReaderGetTsdbRanges(pReader, tsdbTyps[j]);
     if (ppRanges == NULL) continue;
-    tsdbSnapRangeArrayDestroy(ppRanges);
+    tsdbTFileSetRangeArrayDestroy(ppRanges);
   }
 }
 
@@ -170,6 +203,10 @@ void vnodeSnapReaderClose(SVSnapReader *pReader) {
     tsdbSnapReaderClose(&pReader->pTsdbReader);
   }
 
+  if (pReader->pTsdbRAWReader) {
+    tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
+  }
+
   if (pReader->pMetaReader) {
     metaSnapReaderClose(&pReader->pMetaReader);
   }
@@ -285,6 +322,28 @@ int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData)
     }
   }
 
+  if (!pReader->tsdbRAWDone) {
+    // open if not
+    if (pReader->pTsdbRAWReader == NULL) {
+      ASSERT(pReader->sver == 0);
+      code = tsdbSnapRAWReaderOpen(pReader->pVnode->pTsdb, pReader->ever, SNAP_DATA_RAW, &pReader->pTsdbRAWReader);
+      if (code) goto _err;
+    }
+
+    code = tsdbSnapRAWRead(pReader->pTsdbRAWReader, ppData);
+    if (code) {
+      goto _err;
+    } else {
+      if (*ppData) {
+        goto _exit;
+      } else {
+        pReader->tsdbRAWDone = 1;
+        code = tsdbSnapRAWReaderClose(&pReader->pTsdbRAWReader);
+        if (code) goto _err;
+      }
+    }
+  }
+
   // TQ ================
   vInfo("vgId:%d tq transform start", vgId);
   if (!pReader->tqHandleDone) {
@@ -455,8 +514,10 @@ struct SVSnapWriter {
   // meta
   SMetaSnapWriter *pMetaSnapWriter;
   // tsdb
-  TSnapRangeArray *pRanges;
+  TFileSetRangeArray *pRanges;
   STsdbSnapWriter *pTsdbSnapWriter;
+  // tsdb raw
+  STsdbSnapRAWWriter *pTsdbSnapRAWWriter;
   // tq
   STqSnapWriter      *pTqSnapWriter;
   STqOffsetWriter    *pTqOffsetWriter;
@@ -465,11 +526,11 @@ struct SVSnapWriter {
   SStreamTaskWriter  *pStreamTaskWriter;
   SStreamStateWriter *pStreamStateWriter;
   // rsma
-  TSnapRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
+  TFileSetRangeArray *pRsmaRanges[TSDB_RETENTION_L2];
   SRSmaSnapWriter *pRsmaSnapWriter;
 };
 
-TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
+TFileSetRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t tsdbTyp) {
   ASSERTS(sizeof(pWriter->pRsmaRanges) / sizeof(pWriter->pRsmaRanges[0]) == 2, "Unexpected array size");
   switch (tsdbTyp) {
     case SNAP_DATA_TSDB:
@@ -483,7 +544,7 @@ TSnapRangeArray **vnodeSnapWriterGetTsdbRanges(SVSnapWriter *pWriter, int32_t ts
   }
 }
 
-static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
+static int32_t vnodeSnapWriterDealWithSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *pParam) {
   SVnode *pVnode = pWriter->pVnode;
   int32_t code = -1;
 
@@ -494,7 +555,8 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *
       goto _out;
     }
 
-    TSnapRangeArray **ppRanges = NULL;
+    STsdbRepOpts         tsdbOpts = {0};
+    TFileSetRangeArray **ppRanges = NULL;
     int32_t           offset = 0;
 
     while (offset + sizeof(SSyncTLV) < datHead->len) {
@@ -502,16 +564,34 @@ static int32_t vnodeSnapWriterDoSnapInfo(SVSnapWriter *pWriter, SSnapshotParam *
       offset += sizeof(SSyncTLV) + subField->len;
       void   *buf = subField->val;
       int32_t bufLen = subField->len;
-      ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
-      if (ppRanges == NULL) {
-        vError("vgId:%d, unexpected subfield type in data of snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
-        goto _out;
-      }
-      if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
-        vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
-        goto _out;
+
+      switch (subField->typ) {
+        case SNAP_DATA_TSDB:
+        case SNAP_DATA_RSMA1:
+        case SNAP_DATA_RSMA2: {
+          ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, subField->typ);
+          if (ppRanges == NULL) {
+            vError("vgId:%d, unexpected subfield type in snapshot param. subtyp:%d", TD_VID(pVnode), subField->typ);
+            goto _out;
+          }
+          if (vnodeExtractSnapInfoDiff(buf, bufLen, ppRanges) < 0) {
+            vError("vgId:%d, failed to get range diff since %s", TD_VID(pVnode), terrstr());
+            goto _out;
+          }
+        } break;
+        case SNAP_DATA_RAW: {
+          if (tDeserializeTsdbRepOpts(buf, bufLen, &tsdbOpts) < 0) {
+            vError("vgId:%d, failed to deserialize tsdb rep opts since %s", TD_VID(pVnode), terrstr());
+            goto _out;
+          }
+        } break;
+        default:
+          vError("vgId:%d, unexpected subfield type of snap info. typ:%d", TD_VID(pVnode), subField->typ);
+          goto _out;
       }
     }
+
+    vInfo("vgId:%d, vnode snap writer supported tsdb rep of format:%d", TD_VID(pVnode), tsdbOpts.format);
   }
 
   code = 0;
@@ -558,7 +638,7 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, SSnapshotParam *pParam, SVSnapWriter
   pWriter->commitID = ++pVnode->state.commitID;
 
   // snapshot info
-  if (vnodeSnapWriterDoSnapInfo(pWriter, pParam) < 0) {
+  if (vnodeSnapWriterDealWithSnapInfo(pWriter, pParam) < 0) {
     goto _err;
   }
 
@@ -576,9 +656,9 @@ _err:
 static void vnodeSnapWriterDestroyTsdbRanges(SVSnapWriter *pWriter) {
   int32_t tsdbTyps[TSDB_RETENTION_MAX] = {SNAP_DATA_TSDB, SNAP_DATA_RSMA1, SNAP_DATA_RSMA2};
   for (int32_t j = 0; j < TSDB_RETENTION_MAX; ++j) {
-    TSnapRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]);
+    TFileSetRangeArray **ppRanges = vnodeSnapWriterGetTsdbRanges(pWriter, tsdbTyps[j]);
     if (ppRanges == NULL) continue;
-    tsdbSnapRangeArrayDestroy(ppRanges);
+    tsdbTFileSetRangeArrayDestroy(ppRanges);
   }
 }
 
@@ -593,6 +673,10 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter);
   }
 
+  if (pWriter->pTsdbSnapRAWWriter) {
+    tsdbSnapRAWWriterPrepareClose(pWriter->pTsdbSnapRAWWriter);
+  }
+
   if (pWriter->pRsmaSnapWriter) {
     rsmaSnapWriterPrepareClose(pWriter->pRsmaSnapWriter);
   }
@@ -629,6 +713,11 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *
     if (code) goto _exit;
   }
 
+  if (pWriter->pTsdbSnapRAWWriter) {
+    code = tsdbSnapRAWWriterClose(&pWriter->pTsdbSnapRAWWriter, rollback);
+    if (code) goto _exit;
+  }
+
   if (pWriter->pTqSnapWriter) {
     code = tqSnapWriterClose(&pWriter->pTqSnapWriter, rollback);
     if (code) goto _exit;
@@ -752,6 +841,17 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) {
       code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pHdr);
       if (code) goto _err;
     } break;
+    case SNAP_DATA_RAW: {
+      // tsdb
+      if (pWriter->pTsdbSnapRAWWriter == NULL) {
+        ASSERT(pWriter->sver == 0);
+        code = tsdbSnapRAWWriterOpen(pVnode->pTsdb, pWriter->ever, &pWriter->pTsdbSnapRAWWriter);
+        if (code) goto _err;
+      }
+
+      code = tsdbSnapRAWWrite(pWriter->pTsdbSnapRAWWriter, pHdr);
+      if (code) goto _err;
+    } break;
     case SNAP_DATA_TQ_HANDLE: {
       // tq handle
       if (pWriter->pTqSnapWriter == NULL) {

source/dnode/vnode/src/vnd/vnodeSync.c

@@ -804,7 +804,7 @@ int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnap) {
   }
 
   if (pSnap->type == TDMT_SYNC_PREP_SNAPSHOT || pSnap->type == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    code = tsdbSnapGetDetails(pVnode, pSnap);
+    code = tsdbSnapPrepDescription(pVnode, pSnap);
   }
   return code;
 }

source/libs/executor/src/cachescanoperator.c

@@ -352,6 +352,7 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
 
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i);
+    bool found = false;
     for (int32_t j = 0; j < pWrapper->nCols; ++j) {
       /*      if (pColMatch->colId == pWrapper->pSchema[j].colId && pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
         (*pSlotIds)[pColMatch->dstSlotId] = -1;
@@ -361,9 +362,14 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
       if (pColMatch->colId == pWrapper->pSchema[j].colId) {
         (*pSlotIds)[i] = j;
         (*pDstSlotIds)[i] = pColMatch->dstSlotId;
+        found = true;
         break;
       }
     }
+    if (!found) {
+      (*pSlotIds)[i] = -1;
+      (*pDstSlotIds)[i] = pColMatch->dstSlotId;
+    }
   }
 
   return TSDB_CODE_SUCCESS;

source/libs/executor/src/dataInserter.c

@@ -189,7 +189,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
   }
 
   int64_t lastTs = TSKEY_MIN;
-  bool    ignoreRow = false;
+  bool    updateLastRow = false;
   bool    disorderTs = false;
 
   for (int32_t j = 0; j < rows; ++j) {  // iterate by row
@@ -249,7 +249,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
             } else {
               if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
                 if (*(int64_t*)var == lastTs) {
-                  ignoreRow = true;
+                  updateLastRow = true;
                 } else if (*(int64_t*)var < lastTs) {
                   disorderTs = true;
                 } else {
@@ -269,15 +269,6 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
           }
           break;
       }
-
-      if (ignoreRow) {
-        break;
-      }
-    }
-
-    if (ignoreRow) {
-      ignoreRow = false;
-      continue;
     }
 
     SRow* pRow = NULL;
@@ -285,7 +276,14 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp
       tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
       goto _end;
     }
-    taosArrayPush(tbData.aRowP, &pRow);
+    if (updateLastRow) {
+      updateLastRow = false;
+      SRow** lastRow = taosArrayPop(tbData.aRowP);
+      tRowDestroy(*lastRow);
+      taosArrayPush(tbData.aRowP, &pRow);
+    } else {
+      taosArrayPush(tbData.aRowP, &pRow);
+    }
   }
 
   if (disorderTs) {

source/libs/executor/src/streameventwindowoperator.c

@@ -195,6 +195,10 @@ int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pW
       pWinInfo->pWinFlag->endFlag = ends[i];
     } else if (pWin->ekey == pTsData[i]) {
       pWinInfo->pWinFlag->endFlag |= ends[i];
+    } else {
+      *pRebuild = true;
+      pWinInfo->pWinFlag->endFlag |= ends[i];
+      return i + 1 - start;
     }
 
     memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));
@@ -319,6 +323,9 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
       tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));
       doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin);
       releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);
+      SSessionKey tmpSeInfo = {0};
+      getSessionHashKey(&curWin.winInfo.sessionWin, &tmpSeInfo);
+      tSimpleHashPut(pStDeleted, &tmpSeInfo, sizeof(SSessionKey), NULL, 0);
       continue;
     }
     code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,

source/libs/geometry/src/geomFunc.c

@@ -67,15 +67,19 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) {
     return TSDB_CODE_SUCCESS;
   }
 
-  // make input as a zero ending string
-  char *end = varDataVal(input) + varDataLen(input);
-  char endValue = *end;
-  *end = 0; 
-
+  char *inputGeom = NULL;
   unsigned char *outputGeom = NULL;
   size_t size = 0;
 
-  code = doGeomFromText(varDataVal(input), &outputGeom, &size);
+  // make a zero ending string
+  inputGeom = taosMemoryCalloc(1, varDataLen(input) + 1);
+  if (inputGeom == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  memcpy(inputGeom, varDataVal(input), varDataLen(input));
+
+  code = doGeomFromText(inputGeom, &outputGeom, &size);
   if (code != TSDB_CODE_SUCCESS) {
     goto _exit;
   }
@@ -92,8 +96,7 @@ int32_t doGeomFromTextFunc(const char *input, unsigned char **output) {
 
 _exit:
   geosFreeBuffer(outputGeom);
-
-  *end = endValue;  //recover the input string
+  geosFreeBuffer(inputGeom);
 
   return code;
 }

source/libs/parser/src/parInsertSql.c

@@ -507,7 +507,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_TINYINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z);
       } else if (!IS_VALID_TINYINT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z);
@@ -518,7 +519,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z);
       } else if (uv > UINT8_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z);
@@ -528,7 +530,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z);
       } else if (!IS_VALID_SMALLINT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z);
@@ -538,7 +541,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z);
       } else if (uv > UINT16_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z);
@@ -548,7 +552,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_INT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z);
       } else if (!IS_VALID_INT(iv)) {
         return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z);
@@ -558,7 +563,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z);
       } else if (uv > UINT32_MAX) {
         return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z);
@@ -568,7 +574,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_BIGINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) {
+      code = toIntegerEx(pToken->z, pToken->n, pToken->type, &iv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z);
       }
       val->i64 = iv;
@@ -576,7 +583,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
     }
 
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) {
+      code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &uv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z);
       }
       *(uint64_t*)(&val->i64) = uv;
@@ -585,11 +593,11 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
 
     case TSDB_DATA_TYPE_FLOAT: {
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
-          isnan(dv)) {
+      if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
       *(float*)(&val->i64) = dv;
@@ -598,8 +606,9 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema,
 
     case TSDB_DATA_TYPE_DOUBLE: {
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
-        return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
+      code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
+        return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z);
       }
       if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z);
@@ -700,7 +709,8 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStm
 
 static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, SSchema* pTagSchema, SToken* pToken,
                              SArray* pTagName, SArray* pTagVals, STag** pTag) {
-  if (!isNullValue(pTagSchema->type, pToken)) {
+  bool isNull = isNullValue(pTagSchema->type, pToken);
+  if (!isNull) {
     taosArrayPush(pTagName, pTagSchema->name);
   }
 
@@ -709,13 +719,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStm
       return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z);
     }
 
-    if (isNullValue(pTagSchema->type, pToken)) {
+    if (isNull) {
       return tTagNew(pTagVals, 1, true, pTag);
     } else {
       return parseJsontoTagData(pToken->z, pTagVals, pTag, &pCxt->msg);
     }
   }
 
+  if (isNull) return 0;
+
   STagVal val = {0};
   int32_t code =
       parseTagToken(ppSql, pToken, pTagSchema, pStmt->pTableMeta->tableInfo.precision, &val, &pCxt->msg);
@@ -1428,7 +1440,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_TINYINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z);
       } else if (!IS_VALID_TINYINT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z);
@@ -1436,7 +1449,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_UTINYINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z);
       } else if (pVal->value.val > UINT8_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z);
@@ -1444,7 +1458,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_SMALLINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z);
       } else if (!IS_VALID_SMALLINT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z);
@@ -1452,7 +1467,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_USMALLINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z);
       } else if (pVal->value.val > UINT16_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z);
@@ -1460,7 +1476,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_INT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z);
       } else if (!IS_VALID_INT(pVal->value.val)) {
         return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z);
@@ -1468,7 +1485,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_UINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z);
       } else if (pVal->value.val > UINT32_MAX) {
         return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z);
@@ -1476,25 +1494,26 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_BIGINT: {
-      if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z);
       }
       break;
     }
     case TSDB_DATA_TYPE_UBIGINT: {
-      if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) {
+      int32_t code = toUIntegerEx(pToken->z, pToken->n, pToken->type, &pVal->value.val);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z);
       }
       break;
     }
     case TSDB_DATA_TYPE_FLOAT: {
-      char*  endptr = NULL;
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
+      int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) ||
-          isnan(dv)) {
+      if (dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
       float f = dv;
@@ -1502,12 +1521,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql,
       break;
     }
     case TSDB_DATA_TYPE_DOUBLE: {
-      char*  endptr = NULL;
       double dv;
-      if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) {
-        return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
+      int32_t code = toDoubleEx(pToken->z, pToken->n, pToken->type, &dv);
+      if (TSDB_CODE_SUCCESS != code) {
+        return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z);
       }
-      if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) {
+      if (isinf(dv) || isnan(dv)) {
         return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z);
       }
       pVal->value.val = *(int64_t*)&dv;

source/libs/parser/src/parTokenizer.c

@@ -609,6 +609,11 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
         break;
       }
 
+      // support float with no decimal part after the decimal point
+      if (z[i] == '.' && seg == 1) {
+        *tokenId = TK_NK_FLOAT;
+        i++;
+      }
       if ((z[i] == 'e' || z[i] == 'E') &&
           (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) {
         i += 2;
@@ -751,7 +756,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr, bool* pIgnoreC
     // support parse the -/+number format
     if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) {
       len = tGetToken(&str[*i + t0.n], &type);
-      if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) {
+      if (type == TK_NK_INTEGER || type == TK_NK_FLOAT || type == TK_NK_BIN || type == TK_NK_HEX) {
         t0.type = type;
         t0.n += len;
       }

source/libs/parser/src/parTranslater.c

@@ -4622,11 +4622,15 @@ static int32_t addOrderByPrimaryKeyToQueryImpl(STranslateContext* pCxt, SNode* p
     return TSDB_CODE_OUT_OF_MEMORY;
   }
   ((SExprNode*)pOrderByExpr->pExpr)->orderAlias = true;
-  NODES_DESTORY_LIST(*pOrderByList);
+  // NODES_DESTORY_LIST(*pOrderByList);
   return nodesListMakeStrictAppend(pOrderByList, (SNode*)pOrderByExpr);
 }
 
 static int32_t addOrderByPrimaryKeyToQuery(STranslateContext* pCxt, SNode* pPrimaryKeyExpr, SNode* pStmt) {
+  SNodeList* pOrederList = ((SSelectStmt*)pStmt)->pOrderByList;
+  if (pOrederList && pOrederList->length > 0) {
+    return TSDB_CODE_SUCCESS;
+  }
   if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
     return addOrderByPrimaryKeyToQueryImpl(pCxt, pPrimaryKeyExpr, &((SSelectStmt*)pStmt)->pOrderByList);
   }
@@ -7388,6 +7392,13 @@ static int32_t subtableExprHasColumnOrPseudoColumn(SNode* pNode) {
 
 static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) {
   SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
+  if ( (SRealTableNode*)pSelect->pFromTable && ((SRealTableNode*)pSelect->pFromTable)->pMeta
+    && TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType
+    && !hasPartitionByTbname(pSelect->pPartitionByList)
+    && pSelect->pWindow != NULL && pSelect->pWindow->type == QUERY_NODE_EVENT_WINDOW) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
+                                   "Event window for stream on super table must patitioned by table name");
+  }
   if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type ||
       !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList ||
       crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) {

source/libs/planner/src/planSpliter.c

@@ -1034,6 +1034,7 @@ static SNode* stbSplCreateColumnNode(SExprNode* pExpr) {
     strcpy(pCol->colName, pExpr->aliasName);
   }
   strcpy(pCol->node.aliasName, pExpr->aliasName);
+  strcpy(pCol->node.userAlias, pExpr->userAlias);
   pCol->node.resType = pExpr->resType;
   return (SNode*)pCol;
 }
@@ -1063,8 +1064,9 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets,
     SNode*            pTarget = NULL;
     bool              found = false;
     FOREACH(pTarget, pTargets) {
-      if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) ||
-          (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) {
+      if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget))
+        // || (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))
+        ) {
         code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget));
         if (TSDB_CODE_SUCCESS != code) {
           break;

source/libs/sync/inc/syncReplication.h

@@ -56,6 +56,10 @@ int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode);
 
 int32_t syncNodeSendAppendEntries(SSyncNode* pNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg);
 
+int32_t syncSnapSendMsg(SSyncSnapshotSender* pSender, int32_t seq, void* pBlock, int32_t len, int32_t typ);
+int32_t syncSnapSendRsp(SSyncSnapshotReceiver* pReceiver, SyncSnapshotSend* pMsg, void* pBlock, int32_t len,
+                        int32_t typ, int32_t code);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/sync/src/syncSnapshot.c

@@ -23,8 +23,6 @@
 #include "syncReplication.h"
 #include "syncUtil.h"
 
-int32_t syncSnapSendMsg(SSyncSnapshotSender *pSender, int32_t seq, void *pBlock, int32_t len, int32_t typ);
-
 static void syncSnapBufferReset(SSyncSnapBuffer *pBuf) {
   taosThreadMutexLock(&pBuf->mutex);
   for (int64_t i = pBuf->start; i < pBuf->end; ++i) {
@@ -123,6 +121,11 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
   if (pSender->pSndBuf) {
     syncSnapBufferDestroy(&pSender->pSndBuf);
   }
+
+  if (pSender->snapshotParam.data) {
+    taosMemoryFree(pSender->snapshotParam.data);
+    pSender->snapshotParam.data = NULL;
+  }
   // free sender
   taosMemoryFree(pSender);
 }
@@ -153,7 +156,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
   pSender->lastSendTime = taosGetTimestampMs();
   pSender->finish = false;
 
-  // Get full snapshot info
+  // Get snapshot info
   SSyncNode *pSyncNode = pSender->pSyncNode;
   SSnapshot  snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT};
   if (pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo) != 0) {
@@ -161,11 +164,10 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) {
     goto _out;
   }
 
-  int dataLen = 0;
   void   *pData = snapInfo.data;
-  int32_t type = 0;
+  int32_t type = (pData) ? snapInfo.type : 0;
+  int32_t dataLen = 0;
   if (pData) {
-    type = snapInfo.type;
     SSyncTLV *datHead = pData;
     if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT) {
       sSError(pSender, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
@@ -347,9 +349,6 @@ _out:;
   return code;
 }
 
-// return 0, start ok
-// return 1, last snapshot finish ok
-// return -1, error
 int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
   SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, pDestId);
   if (pSender == NULL) {
@@ -380,6 +379,7 @@ int32_t syncNodeStartSnapshot(SSyncNode *pSyncNode, SRaftId *pDestId) {
   return 0;
 }
 
+// receiver
 SSyncSnapshotReceiver *snapshotReceiverCreate(SSyncNode *pSyncNode, SRaftId fromId) {
   bool condition = (pSyncNode->pFsm->FpSnapshotStartWrite != NULL) && (pSyncNode->pFsm->FpSnapshotStopWrite != NULL) &&
                    (pSyncNode->pFsm->FpSnapshotDoWrite != NULL);
@@ -509,8 +509,6 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *p
   sRInfo(pReceiver, "snapshot receiver start, from dnode:%d.", DID(&pReceiver->fromId));
 }
 
-// just set start = false
-// FpSnapshotStopWrite should not be called
 void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
   sRDebug(pReceiver, "snapshot receiver stop, not apply, writer:%p", pReceiver->pWriter);
 
@@ -531,7 +529,6 @@ void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) {
   syncSnapBufferReset(pReceiver->pRcvBuf);
 }
 
-// when recv last snapshot block, apply data into snapshot
 static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
   int32_t code = 0;
   if (pReceiver->pWriter != NULL) {
@@ -590,8 +587,6 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap
   return 0;
 }
 
-// apply data block
-// update progress
 static int32_t snapshotReceiverGotData(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg) {
   if (pMsg->seq != pReceiver->ack + 1) {
     sRError(pReceiver, "snapshot receiver invalid seq, ack:%d seq:%d", pReceiver->ack, pMsg->seq);
@@ -644,6 +639,50 @@ SyncIndex syncNodeGetSnapBeginIndex(SSyncNode *ths) {
   return snapStart;
 }
 
+static int32_t syncSnapReceiverExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotReceiver *pReceiver,
+                                             SyncSnapshotSend *pMsg, SSnapshot *pInfo) {
+  ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT);
+  int32_t code = 0;
+
+  // copy snap info from leader
+  void *data = taosMemoryCalloc(1, pMsg->dataLen);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = terrno;
+    goto _out;
+  }
+  pInfo->data = data;
+  data = NULL;
+  memcpy(pInfo->data, pMsg->data, pMsg->dataLen);
+
+  // exchange snap info
+  pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, pInfo);
+  SSyncTLV *datHead = pInfo->data;
+  if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
+    sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
+    code = TSDB_CODE_INVALID_DATA_FMT;
+    goto _out;
+  }
+  int32_t dataLen = sizeof(SSyncTLV) + datHead->len;
+
+  // save exchanged snap info
+  SSnapshotParam *pParam = &pReceiver->snapshotParam;
+  data = taosMemoryRealloc(pParam->data, dataLen);
+  if (data == NULL) {
+    sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId,
+           strerror(errno), dataLen);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    code = terrno;
+    goto _out;
+  }
+  pParam->data = data;
+  data = NULL;
+  memcpy(pParam->data, pInfo->data, dataLen);
+
+_out:
+  return code;
+}
+
 static int32_t syncNodeOnSnapshotPrep(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) {
   SSyncSnapshotReceiver *pReceiver = pSyncNode->pNewNodeReceiver;
   int64_t                timeNow = taosGetTimestampMs();
@@ -686,77 +725,27 @@ _START_RECEIVER:
     snapshotReceiverStop(pReceiver);
   }
 
-  snapshotReceiverStart(pReceiver, pMsg);  // set start-time same with sender
+  snapshotReceiverStart(pReceiver, pMsg);
 
-_SEND_REPLY:
-    // build msg
-    ;  // make complier happy
+_SEND_REPLY:;
 
   SSnapshot snapInfo = {.type = TDMT_SYNC_PREP_SNAPSHOT_REPLY};
   int32_t   dataLen = 0;
-  if (pMsg->dataLen > 0) {
-    void *data = taosMemoryCalloc(1, pMsg->dataLen);
-    if (data == NULL) {
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      code = terrno;
+  if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT) {
+    if (syncSnapReceiverExchgSnapInfo(pSyncNode, pReceiver, pMsg, &snapInfo) != 0) {
       goto _out;
     }
-    memcpy(data, pMsg->data, pMsg->dataLen);
-    snapInfo.data = data;
-    data = NULL;
-    pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapInfo);
-
     SSyncTLV *datHead = snapInfo.data;
-    if (datHead->typ != TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-      sRError(pReceiver, "unexpected data typ in data of snapshot info. typ: %d", datHead->typ);
-      code = TSDB_CODE_INVALID_DATA_FMT;
-      goto _out;
-    }
     dataLen = sizeof(SSyncTLV) + datHead->len;
   }
 
-  SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSendRsp(&rpcMsg, dataLen, pSyncNode->vgId) != 0) {
-    sRError(pReceiver, "snapshot receiver failed to build resp since %s", terrstr());
+  // send response
+  int32_t type = (snapInfo.data) ? snapInfo.type : 0;
+  if (syncSnapSendRsp(pReceiver, pMsg, snapInfo.data, dataLen, type, code) != 0) {
     code = terrno;
     goto _out;
   }
 
-  SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
-  pRspMsg->srcId = pSyncNode->myRaftId;
-  pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
-  pRspMsg->lastIndex = pMsg->lastIndex;
-  pRspMsg->lastTerm = pMsg->lastTerm;
-  pRspMsg->startTime = pMsg->startTime;
-  pRspMsg->ack = pMsg->seq;  // receiver maybe already closed
-  pRspMsg->code = code;
-  pRspMsg->snapBeginIndex = syncNodeGetSnapBeginIndex(pSyncNode);
-
-  if (snapInfo.data) {
-    pRspMsg->payloadType = snapInfo.type;
-    memcpy(pRspMsg->data, snapInfo.data, dataLen);
-
-    // save snapshot info
-    SSnapshotParam *pParam = &pReceiver->snapshotParam;
-    void           *data = taosMemoryRealloc(pParam->data, dataLen);
-    if (data == NULL) {
-      sError("vgId:%d, failed to realloc memory for snapshot prep due to %s. dataLen:%d", pSyncNode->vgId,
-             strerror(errno), dataLen);
-      terrno = TSDB_CODE_OUT_OF_MEMORY;
-      code = terrno;
-      goto _out;
-    }
-    pParam->data = data;
-    memcpy(pParam->data, snapInfo.data, dataLen);
-  }
-
-  // send msg
-  if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
-    sRError(pReceiver, "failed to send resp since %s", terrstr());
-    code = terrno;
-  }
-
 _out:
   if (snapInfo.data) {
     taosMemoryFree(snapInfo.data);
@@ -793,38 +782,20 @@ _SEND_REPLY:
     code = terrno;
   }
 
-  // build msg
-  SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId) != 0) {
-    sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr());
-    return -1;
-  }
-
-  SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
-  pRspMsg->srcId = pSyncNode->myRaftId;
-  pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
-  pRspMsg->lastIndex = pMsg->lastIndex;
-  pRspMsg->lastTerm = pMsg->lastTerm;
-  pRspMsg->startTime = pMsg->startTime;
-  pRspMsg->ack = pReceiver->ack;  // receiver maybe already closed
-  pRspMsg->code = code;
-  pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start;
-
-  // send msg
-  if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
-    sRError(pReceiver, "failed to send snapshot receiver resp since %s", terrstr());
+  // send response
+  if (syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code) != 0) {
     return -1;
   }
 
   return code;
 }
 
-static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, int32_t code) {
+int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pMsg, void *pBlock, int32_t blockLen,
+                        int32_t type, int32_t code) {
   SSyncNode *pSyncNode = pReceiver->pSyncNode;
   // build msg
   SRpcMsg rpcMsg = {0};
-  if (syncBuildSnapshotSendRsp(&rpcMsg, 0, pSyncNode->vgId)) {
+  if (syncBuildSnapshotSendRsp(&rpcMsg, blockLen, pSyncNode->vgId)) {
     sRError(pReceiver, "failed to build snapshot receiver resp since %s", terrstr());
     return -1;
   }
@@ -832,13 +803,18 @@ static int32_t syncSnapSendRsp(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSen
   SyncSnapshotRsp *pRspMsg = rpcMsg.pCont;
   pRspMsg->srcId = pSyncNode->myRaftId;
   pRspMsg->destId = pMsg->srcId;
-  pRspMsg->term = raftStoreGetTerm(pSyncNode);
+  pRspMsg->term = pMsg->term;
   pRspMsg->lastIndex = pMsg->lastIndex;
   pRspMsg->lastTerm = pMsg->lastTerm;
   pRspMsg->startTime = pMsg->startTime;
   pRspMsg->ack = pMsg->seq;
   pRspMsg->code = code;
   pRspMsg->snapBeginIndex = pReceiver->snapshotParam.start;
+  pRspMsg->payloadType = type;
+
+  if (pBlock != NULL && blockLen > 0) {
+    memcpy(pRspMsg->data, pBlock, blockLen);
+  }
 
   // send msg
   if (syncNodeSendMsgById(&pRspMsg->destId, pSyncNode, &rpcMsg) != 0) {
@@ -872,7 +848,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
     ppMsg[0] = NULL;
     pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end);
   } else if (pMsg->seq < pRcvBuf->start) {
-    syncSnapSendRsp(pReceiver, pMsg, code);
+    syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, code);
     goto _out;
   }
 
@@ -892,7 +868,7 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot
       }
     }
     pRcvBuf->start = seq + 1;
-    syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], code);
+    syncSnapSendRsp(pReceiver, pRcvBuf->entries[seq % pRcvBuf->size], NULL, 0, 0, code);
     pRcvBuf->entryDeleteCb(pRcvBuf->entries[seq % pRcvBuf->size]);
     pRcvBuf->entries[seq % pRcvBuf->size] = NULL;
     if (code) goto _out;
@@ -915,7 +891,7 @@ static int32_t syncNodeOnSnapshotReceive(SSyncNode *pSyncNode, SyncSnapshotSend
   if (snapshotReceiverSignatureCmp(pReceiver, pMsg) != 0) {
     terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE;
     sRError(pReceiver, "failed to receive snapshot data since %s.", terrstr());
-    return syncSnapSendRsp(pReceiver, pMsg, terrno);
+    return syncSnapSendRsp(pReceiver, pMsg, NULL, 0, 0, terrno);
   }
 
   return syncSnapBufferRecv(pReceiver, ppMsg);
@@ -971,26 +947,6 @@ _SEND_REPLY:;
   return code;
 }
 
-// receiver on message
-//
-// condition 1, recv SYNC_SNAPSHOT_SEQ_PREP
-//              if receiver already start
-//                    if sender.start-time > receiver.start-time, restart receiver(reply snapshot start)
-//                    if sender.start-time = receiver.start-time, maybe duplicate msg
-//                    if sender.start-time < receiver.start-time, ignore
-//              else
-//                    waiting for clock match
-//                    start receiver(reply snapshot start)
-//
-// condition 2, recv SYNC_SNAPSHOT_SEQ_BEGIN
-//              a. create writer with <begin, end>
-//
-// condition 3, recv SYNC_SNAPSHOT_SEQ_END, finish receiver(apply snapshot data, update commit index, maybe reconfig)
-//
-// condition 4, recv SYNC_SNAPSHOT_SEQ_FORCE_CLOSE, force close
-//
-// condition 5, got data, update ack
-//
 int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
   SyncSnapshotSend **ppMsg = (SyncSnapshotSend **)&pRpcMsg->pCont;
   SyncSnapshotSend      *pMsg = ppMsg[0];
@@ -1074,6 +1030,32 @@ _out:;
   return code;
 }
 
+static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
+  ASSERT(pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY);
+
+  SSyncTLV *datHead = (void *)pMsg->data;
+  if (datHead->typ != pMsg->payloadType) {
+    sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ);
+    terrno = TSDB_CODE_INVALID_DATA_FMT;
+    return -1;
+  }
+  int32_t dataLen = sizeof(SSyncTLV) + datHead->len;
+
+  SSnapshotParam *pParam = &pSender->snapshotParam;
+  void           *data = taosMemoryRealloc(pParam->data, dataLen);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  memcpy(data, pMsg->data, dataLen);
+
+  pParam->data = data;
+  data = NULL;
+  sSInfo(pSender, "data of snapshot param. len: %d", datHead->len);
+  return 0;
+}
+
+// sender
 static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) {
   SSnapshot snapshot = {0};
   pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
@@ -1090,14 +1072,9 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend
 
   // start reader
   if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) {
-    SSyncTLV *datHead = (void *)pMsg->data;
-    if (datHead->typ != pMsg->payloadType) {
-      sSError(pSender, "unexpected data type in data of SyncSnapshotRsp. typ: %d", datHead->typ);
-      terrno = TSDB_CODE_INVALID_DATA_FMT;
+    if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) {
       return -1;
     }
-    pSender->snapshotParam.data = (void *)pMsg->data;
-    sSInfo(pSender, "data of snapshot param. len: %d", datHead->len);
   }
 
   int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader);
@@ -1131,7 +1108,7 @@ static int32_t syncSnapBufferSend(SSyncSnapshotSender *pSender, SyncSnapshotRsp
     goto _out;
   }
 
-  if (pSender->pReader == NULL || pSender->finish) {
+  if (pSender->pReader == NULL || pSender->finish || !snapshotSenderIsStart(pSender)) {
     code = terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
     goto _out;
   }
@@ -1182,12 +1159,6 @@ _out:
   return code;
 }
 
-// sender on message
-//
-// condition 1 sender receives SYNC_SNAPSHOT_SEQ_END, close sender
-// condition 2 sender receives ack, set seq = ack + 1, send msg from seq
-// condition 3 sender receives error msg, just print error log
-//
 int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) {
   SyncSnapshotRsp **ppMsg = (SyncSnapshotRsp **)&pRpcMsg->pCont;
   SyncSnapshotRsp  *pMsg = ppMsg[0];

source/libs/sync/src/syncUtil.c

@@ -487,3 +487,13 @@ void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteRepl
   sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId),
          pMsg->term, pMsg->voteGranted, s);
 }
+
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size) {
+  void* data = taosMemoryRealloc(pSnap->data, size);
+  if (data == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
+  }
+  pSnap->data = data;
+  return 0;
+}

source/libs/transport/src/thttp.c

@@ -34,6 +34,7 @@ typedef struct SHttpModule {
   SAsyncPool* asyncPool;
   TdThread    thread;
   SHashObj*   connStatusTable;
+  int8_t      quit;
 } SHttpModule;
 
 typedef struct SHttpMsg {
@@ -166,7 +167,7 @@ _OVER:
 static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
   uint32_t ip = taosGetIpv4FromFqdn(server);
   if (ip == 0xffffffff) {
-    tError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr());
+    tError("http-report failed to resolving domain names: %s", server);
     return -1;
   }
   char buf[128] = {0};
@@ -190,19 +191,40 @@ static void httpDestroyMsg(SHttpMsg* msg) {
   taosMemoryFree(msg->cont);
   taosMemoryFree(msg);
 }
+
+static void httpMayDiscardMsg(SHttpModule* http, SAsyncItem* item) {
+  SHttpMsg *msg = NULL, *quitMsg = NULL;
+  if (atomic_load_8(&http->quit) == 0) {
+    return;
+  }
+
+  while (!QUEUE_IS_EMPTY(&item->qmsg)) {
+    queue* h = QUEUE_HEAD(&item->qmsg);
+    QUEUE_REMOVE(h);
+    msg = QUEUE_DATA(h, SHttpMsg, q);
+    if (!msg->quit) {
+      httpDestroyMsg(msg);
+    } else {
+      quitMsg = msg;
+    }
+  }
+  if (quitMsg != NULL) {
+    QUEUE_PUSH(&item->qmsg, &quitMsg->q);
+  }
+}
 static void httpAsyncCb(uv_async_t* handle) {
   SAsyncItem*  item = handle->data;
   SHttpModule* http = item->pThrd;
 
   SHttpMsg *msg = NULL, *quitMsg = NULL;
-
-  queue wq;
+  queue     wq;
   QUEUE_INIT(&wq);
 
   static int32_t BATCH_SIZE = 5;
   int32_t        count = 0;
 
   taosThreadMutexLock(&item->mtx);
+  httpMayDiscardMsg(http, item);
 
   while (!QUEUE_IS_EMPTY(&item->qmsg) && count++ < BATCH_SIZE) {
     queue* h = QUEUE_HEAD(&item->qmsg);
@@ -497,9 +519,10 @@ static void transHttpDestroyHandle(void* handle) { taosMemoryFree(handle); }
 static void transHttpEnvInit() {
   httpRefMgt = taosOpenRef(1, transHttpDestroyHandle);
 
-  SHttpModule* http = taosMemoryMalloc(sizeof(SHttpModule));
+  SHttpModule* http = taosMemoryCalloc(1, sizeof(SHttpModule));
   http->loop = taosMemoryMalloc(sizeof(uv_loop_t));
   http->connStatusTable = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
+  http->quit = 0;
 
   uv_loop_init(http->loop);
 
@@ -526,6 +549,8 @@ void transHttpEnvDestroy() {
     return;
   }
   SHttpModule* load = taosAcquireRef(httpRefMgt, httpRef);
+
+  atomic_store_8(&load->quit, 1);
   httpSendQuit();
   taosThreadJoin(load->thread, NULL);
 

source/os/src/osFile.c

@@ -348,15 +348,15 @@ HANDLE taosOpenFileNotStream(const char *path, int32_t tdFileOptions) {
   if (h != INVALID_HANDLE_VALUE && (tdFileOptions & TD_FILE_APPEND) && (tdFileOptions & TD_FILE_WRITE)) {
     SetFilePointer(h, 0, NULL, FILE_END);
   }
-  if (h == INVALID_HANDLE_VALUE) {
-    DWORD dwError = GetLastError();
-    LPVOID lpMsgBuf;
-    FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError,
-                  0,
-                  (LPTSTR)&lpMsgBuf, 0, NULL);
-    printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf);
-    LocalFree(lpMsgBuf);
-  }
+  // if (h == INVALID_HANDLE_VALUE) {
+  //   DWORD dwError = GetLastError();
+  //   LPVOID lpMsgBuf;
+  //   FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, dwError,
+  //                 0,
+  //                 (LPTSTR)&lpMsgBuf, 0, NULL);
+  //   printf("CreateFile failed with error %d: %s", dwError, (char*)lpMsgBuf);
+  //   LocalFree(lpMsgBuf);
+  // }
   return h;
 }
 

source/os/test/osTests.cpp

@@ -343,6 +343,6 @@ TEST(osTest, osFilePerformance) {
   printf("Test OpenForRead & Close file %d times, cost: %" PRId64 "us\n", TESTTIMES, OpenForReadCloseFileCost);
 }
 
-#endif OSFILE_PERFORMANCE_TEST
+#endif // OSFILE_PERFORMANCE_TEST
 
 #pragma GCC diagnostic pop

tests/docs-examples-test/python.sh

@@ -86,7 +86,7 @@ pip3 install kafka-python
 python3 kafka_example_consumer.py
 
 # 21
-pip3 install taos-ws-py==0.2.6
+pip3 install taos-ws-py==0.3.1
 python3 conn_websocket_pandas.py
 
 # 22

tests/parallel_test/cases.task

@@ -250,6 +250,7 @@ e
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py
 
+,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_double.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py
@@ -1132,6 +1133,9 @@ e
 ,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim
 ,,y,script,./test.sh -f tsim/stream/distributeSession0.sim
 ,,y,script,./test.sh -f tsim/stream/drop_stream.sim
+,,y,script,./test.sh -f tsim/stream/event0.sim
+,,y,script,./test.sh -f tsim/stream/event1.sim
+,,y,script,./test.sh -f tsim/stream/event2.sim
 ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim
 ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim
 ,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim

tests/script/tsim/insert/insert_stb.sim

@@ -66,6 +66,29 @@ if $rows != 1 then
   return -1
 endi
 
+sql insert into st(tbname, ts, f, t) values('ct2',now,20,NULL)('ct3',now,30,NULL)
+sql insert into st(tbname, t, ts, f) values('ct4',NULL, now,20)('ct5',NULL, now,30)
+sql show create table ct2
+sql show create table ct3
+sql show create table ct4
+sql show create table ct5
+sql show tags from ct2
+if $data05 != NULL then
+  return -1
+endi
+sql show tags from ct3
+if $data05 != NULL then
+  return -1
+endi
+sql show tags from ct4
+if $data05 != NULL then
+  return -1
+endi
+sql show tags from ct5
+if $data05 != NULL then
+  return -1
+endi
+
 sql_error insert into d2.st values(now, 1, 1)
 sql_error insert into d2.st(ts, f) values(now, 1);
 sql_error insert into d2.st(ts, f, tbname) values(now, 1);

tests/script/tsim/parser/columnValue_bigint.sim

@@ -17,6 +17,7 @@ sql create table mt_bigint (ts timestamp, c bigint) tags (tagname bigint)
 
 ## case 00: static create table for test tag values
 sql create table st_bigint_0  using mt_bigint tags (NULL)
+sql show create table st_bigint_0
 sql show tags from  st_bigint_0
 if $data05 != NULL then
   return -1
@@ -173,6 +174,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_bigint_16 using mt_bigint tags (NULL)   values (now, NULL)
+sql show create table st_bigint_16
 sql show tags from  st_bigint_16
 if $data05 != NULL then
   return -1

tests/script/tsim/parser/columnValue_bool.sim

@@ -17,6 +17,7 @@ sql create table mt_bool (ts timestamp, c bool) tags (tagname bool)
 
 ## case 00: static create table for test tag values
 sql create table st_bool_0  using mt_bool tags (NULL)
+sql show create table st_bool_0
 sql show tags from st_bool_0
 if $data05 != NULL then
   print ==1== expect: NULL, actually: $data05
@@ -291,6 +292,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_bool_16 using mt_bool tags (NULL)   values (now, NULL)
+sql show create table st_bool_16
 sql show tags from st_bool_16
 if $data05 != NULL then
   print ==33== expect: NULL, actually: $data00

tests/script/tsim/parser/columnValue_float.sim

@@ -17,6 +17,7 @@ sql create table mt_float (ts timestamp, c float) tags (tagname float)
 
 ## case 00: static create table for test tag values
 sql create table st_float_0  using mt_float tags (NULL)
+sql show create table st_float_0
 sql show tags from  st_float_0
 if $data05 != NULL then
   return -1
@@ -299,6 +300,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_float_16 using mt_float tags (NULL)   values (now, NULL)
+sql show create table st_float_16
 sql show tags from  st_float_16
 if $data05 != NULL then
   return -1

tests/script/tsim/parser/columnValue_int.sim

@@ -17,6 +17,7 @@ sql create table mt_int (ts timestamp, c int) tags (tagname int)
 
 ## case 00: static create table for test tag values
 sql create table st_int_0  using mt_int tags (NULL)
+sql show create table st_int_0
 sql show tags from  st_int_0
 if $data05 != NULL then
   return -1
@@ -172,6 +173,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_int_16 using mt_int tags (NULL)   values (now, NULL)
+sql show create table st_int_16
 sql show tags from  st_int_16
 if $data05 != NULL then
   return -1

tests/script/tsim/parser/columnValue_smallint.sim

@@ -20,6 +20,7 @@ sql create table mt_smallint (ts timestamp, c smallint) tags (tagname smallint)
 
 ## case 00: static create table for test tag values
 sql create table st_smallint_0  using mt_smallint tags (NULL)
+sql show create table st_smallint_0
 sql show tags from st_smallint_0
 if $data05 != NULL then
   return -1
@@ -175,6 +176,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_smallint_16 using mt_smallint tags (NULL)   values (now, NULL)
+sql show create table st_smallint_16
 sql show tags from st_smallint_16
 if $data05 != NULL then
   return -1

tests/script/tsim/parser/columnValue_tinyint.sim

@@ -17,6 +17,7 @@ sql create table mt_tinyint (ts timestamp, c tinyint) tags (tagname tinyint)
 
 ## case 00: static create table for test tag values
 sql create table st_tinyint_0  using mt_tinyint tags (NULL)
+sql show create table st_tinyint_0
 sql show tags from  st_tinyint_0
 if $data05 != NULL then
   print expect NULL, actually: $data05
@@ -173,6 +174,7 @@ endi
 
 ## case 02: dynamic create table for test tag values
 sql insert into st_tinyint_16 using mt_tinyint tags (NULL)   values (now, NULL)
+sql show create table st_tinyint_16
 sql show tags from  st_tinyint_16
 if $data05 != NULL then
   return -1

tests/script/tsim/parser/null_char.sim

@@ -13,6 +13,7 @@ sql use $db
 #### case 0: field NULL, or 'NULL'
 sql create table mt1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 binary(8), col6 bool, col7 smallint, col8 tinyint, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag4 bigint, tag5 bool, tag6 float)
 sql create table st1 using mt1 tags (NULL, 'NULL', 100, 1000, 'false', 9.123)
+sql show create table st1
 sql insert into st1 values ('2019-01-01 09:00:00.000', 123, -123, 3.0, 4.0, 'binary', true, 1000, 121, 'nchar')
 sql insert into st1 values ('2019-01-01 09:00:01.000', '456', '456', '3.33', '4.444', 'binary', 'true', '1001', '122', 'nchar')
 sql insert into st1 values ('2019-01-01 09:00:02.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
@@ -117,6 +118,7 @@ if $rows != 0 then
 endi
 
 sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE')
+sql show create table st3
 sql insert into st3 (ts, col1) values(now, 1)
 sql select tag1, tag2, tag3, tag5 from st3
 if $rows != 1 then

tests/system-test/0-others/test_create_same_name_db.py

@@ -0,0 +1,38 @@
+import time
+import os
+import platform
+import taos
+import threading
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+from util.common import *
+
+
+class TDTestCase:
+    """This test case is used to veirfy TD-25762
+    """
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor())
+        self.db_name = "db"
+
+    def run(self):
+        try:
+            # create same name database multiple times
+            for i in range(100):
+                tdLog.debug(f"round {str(i+1)} create database {self.db_name}")
+                tdSql.execute(f"create database {self.db_name}")
+                tdLog.debug(f"round {str(i+1)} drop database {self.db_name}")
+                tdSql.execute(f"drop database {self.db_name}")
+        except Exception as ex:
+            tdLog.exit(str(ex))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/1-insert/insert_double.py

@@ -0,0 +1,136 @@
+import taos
+import sys
+import datetime
+import inspect
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+import random
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        self.database = "db1"
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepare_db(self):
+        tdSql.execute(f"drop database if exists {self.database}")
+        tdSql.execute(f"create database {self.database}")
+        tdSql.execute(f"use {self.database}")
+
+    def test_value(self, table_name, dtype, bits):
+        tdSql.execute(f"drop table if exists {table_name}")
+        tdSql.execute(f"create table {table_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned)")
+
+        tdSql.execute(f"insert into {table_name} values(now, -16, +6)")
+        tdSql.execute(f"insert into {table_name} values(now, 80.99, +0042)")
+        tdSql.execute(f"insert into {table_name} values(now, -0042, +80.99)")
+        tdSql.execute(f"insert into {table_name} values(now, 52.34354, 18.6)")
+        tdSql.execute(f"insert into {table_name} values(now, -12., +3.)")
+        tdSql.execute(f"insert into {table_name} values(now, -0.12, +3.0)")
+        tdSql.execute(f"insert into {table_name} values(now, -2.3e1, +2.324e2)")
+        tdSql.execute(f"insert into {table_name} values(now, -2e1,  +2e2)")
+        tdSql.execute(f"insert into {table_name} values(now, -2.e1, +2.e2)")
+        tdSql.execute(f"insert into {table_name} values(now, -0x40, +0b10000)")
+        tdSql.execute(f"insert into {table_name} values(now, -0b10000, +0x40)")
+
+        # str support
+        tdSql.execute(f"insert into {table_name} values(now, '-16', '+6')")
+        tdSql.execute(f"insert into {table_name} values(now, ' -80.99', ' +0042')")
+        tdSql.execute(f"insert into {table_name} values(now, ' -0042', ' +80.99')")
+        tdSql.execute(f"insert into {table_name} values(now, '52.34354', '18.6')")
+        tdSql.execute(f"insert into {table_name} values(now, '-12.', '+5.')")
+        tdSql.execute(f"insert into {table_name} values(now, '-.12', '+.5')")
+        tdSql.execute(f"insert into {table_name} values(now, '-2.e1', '+2.e2')")
+        tdSql.execute(f"insert into {table_name} values(now, '-2e1',  '+2e2')")
+        tdSql.execute(f"insert into {table_name} values(now, '-2.3e1', '+2.324e2')")
+        tdSql.execute(f"insert into {table_name} values(now, '-0x40', '+0b10010')")
+        tdSql.execute(f"insert into {table_name} values(now, '-0b10010', '+0x40')")
+
+        tdSql.query(f"select * from {table_name}")
+        tdSql.checkRows(22)
+
+        baseval = 2**(bits/2)
+        negval = -baseval + 1.645
+        posval = baseval + 4.323
+        bigval = 2**(bits-1)
+        max_i = bigval - 1
+        min_i = -bigval
+        max_u = 2*bigval - 1
+        min_u = 0
+        print("val:", baseval, negval, posval, max_i)
+
+        tdSql.execute(f"insert into {table_name} values(now, {negval}, {posval})")
+        tdSql.execute(f"insert into {table_name} values(now, -{baseval}, {baseval})")
+        tdSql.execute(f"insert into {table_name} values(now, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {table_name} values(now, {min_i}, {min_u})")
+
+        tdSql.query(f"select * from {table_name}")
+        tdSql.checkRows(26)
+        
+        # error case
+        tdSql.error(f"insert into {table_name} values(now, 0, {max_u+1})")
+        tdSql.error(f"insert into {table_name} values(now, 0, -1)")
+        tdSql.error(f"insert into {table_name} values(now, 0, -2.0)")
+        tdSql.error(f"insert into {table_name} values(now, 0, '-2.0')")
+        tdSql.error(f"insert into {table_name} values(now, {max_i+1}, 0)")
+        tdSql.error(f"insert into {table_name} values(now, {min_i-1}, 0)")
+        tdSql.error(f"insert into {table_name} values(now, '{min_i-1}', 0)")
+
+    def test_tags(self, stable_name, dtype, bits):
+        tdSql.execute(f"create stable {stable_name}(ts timestamp, i1 {dtype}, i2 {dtype} unsigned) tags(id {dtype})")
+
+        baseval = 2**(bits/2)
+        negval = -baseval + 1.645
+        posval = baseval + 4.323
+        bigval = 2**(bits-1)
+        max_i = bigval - 1
+        min_i = -bigval
+        max_u = 2*bigval - 1
+        min_u = 0
+
+        tdSql.execute(f"insert into {stable_name}_1 using {stable_name} tags('{negval}') values(now, {negval}, {posval})")
+        tdSql.execute(f"insert into {stable_name}_2 using {stable_name} tags({posval}) values(now, -{baseval} , {baseval})")
+        tdSql.execute(f"insert into {stable_name}_3 using {stable_name} tags('0x40') values(now, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {stable_name}_4 using {stable_name} tags(0b10000) values(now, {min_i}, {min_u})")
+        
+        tdSql.execute(f"insert into {stable_name}_5 using {stable_name} tags({max_i}) values(now, '{negval}', '{posval}')")
+        tdSql.execute(f"insert into {stable_name}_6 using {stable_name} tags('{min_i}') values(now, '-{baseval}' , '{baseval}')")
+        tdSql.execute(f"insert into {stable_name}_7 using {stable_name} tags(-0x40) values(now, '{max_i}', '{max_u}')")
+        tdSql.execute(f"insert into {stable_name}_8 using {stable_name} tags('-0b10000') values(now, '{min_i}', '{min_u}')")
+
+        tdSql.execute(f"insert into {stable_name}_9 using {stable_name} tags(12.) values(now, {negval}, {posval})")
+        tdSql.execute(f"insert into {stable_name}_10 using {stable_name} tags('-8.3') values(now, -{baseval} , {baseval})")
+        tdSql.execute(f"insert into {stable_name}_11 using {stable_name} tags(2.e1) values(now, {max_i}, {max_u})")
+        tdSql.execute(f"insert into {stable_name}_12 using {stable_name} tags('-2.3e1') values(now, {min_i}, {min_u})")
+
+        tdSql.query(f"select * from {stable_name}")
+        tdSql.checkRows(12)
+
+    def run(self):  # sourcery skip: extract-duplicate-method, remove-redundant-fstring
+        tdSql.prepare(replica = self.replicaVar)
+        self.prepare_db()
+
+        self.test_value("t1", "bigint", 64)
+        self.test_value("t2", "int", 32)
+        self.test_value("t3", "smallint", 16)
+        self.test_value("t4", "tinyint", 8)
+        tdLog.printNoPrefix("==========end case1 run ...............")
+
+        self.test_tags("t_big", "bigint", 64)
+        self.test_tags("t_int", "int", 32)
+        self.test_tags("t_small", "smallint", 16)
+        self.test_tags("t_tiny", "tinyint", 8)
+        tdLog.printNoPrefix("==========end case2 run ...............")
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/1-insert/insert_perf.py

@@ -0,0 +1,146 @@
+import taos
+import sys
+import random
+import time
+import csv
+
+from datetime import datetime
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        
+        self.testcasePath = os.path.split(__file__)[0]
+        self.testcasefilename = os.path.split(__file__)[-1]
+        self.file1 = f"{self.testcasePath}/int.csv"
+        self.file2 = f"{self.testcasePath}/double.csv"
+        self.file3 = f"{self.testcasePath}/d+.csv"
+        self.file4 = f"{self.testcasePath}/uint.csv"
+        self.ts = 1700638570000  # 2023-11-22T07:36:10.000Z
+        self.database = "db1"
+        self.tb1 = "t1"
+        self.tb2 = "t2"
+        self.tb3 = "t3"
+        self.once = 1000
+        tdLog.debug(f"start to excute {__file__}")
+        tdSql.init(conn.cursor(), False)
+
+    def prepare_db(self):
+        tdSql.execute(f"drop database if exists {self.database}")
+        tdSql.execute(f"create database {self.database}")
+        tdSql.execute(f"use {self.database}")
+        tdSql.execute(f"create table {self.tb1} (ts timestamp, i0 bigint , i1 bigint, i2 bigint, i3 bigint, i4 bigint, i5 bigint, i6 bigint, i7 bigint, i8 bigint, i9 bigint)")
+        tdSql.execute(f"create table {self.tb2} (ts timestamp, f0 double, f1 double, f2 double, f3 double, f4 double, f5 double, f6 double, f7 double, f8 double, f9 double)")
+        tdSql.execute(f"create table {self.tb3} (ts timestamp, i0 int unsigned , i1 int unsigned, i2 int unsigned, i3 int unsigned, i4 int unsigned, i5 int unsigned, i6 int unsigned, i7 int unsigned, i8 int unsigned, i9 int unsigned)")
+
+    def make_csv(self, once, intype):
+        filepath = self.file1
+        if intype == 2:
+           filepath = self.file2
+        elif intype == 3:
+           filepath = self.file3
+        elif intype == 4:
+           filepath = self.file4
+
+        f = open(filepath, 'w')
+        with f:
+          writer = csv.writer(f)
+          rows = []
+          for i in range(once):
+            r = []
+            if intype == 1:
+              for k in range(10):
+                  r.append(random.randint(-2147483648, 2147483647))
+            elif intype == 2:
+              for k in range(10):
+                  r.append(random.randint(-2147483648, 2147483646) + random.random())
+            elif intype == 3:
+              for k in range(10):
+                  r.append(random.randint(0, 4294967294) + random.random())
+            else:
+              for k in range(10):
+                  r.append(random.randint(0, 4294967295))
+            rows.append(r)
+          writer.writerows(rows)
+        f.close()
+        print(f"{filepath} ready!")
+
+    def test_insert(self, tbname, qtime, startts, intype, outtype):
+        filepath = self.file1
+        dinfo = "int"
+        if intype == 2:
+           filepath = self.file2
+           dinfo = "double"
+        elif intype == 3:
+           filepath = self.file3
+           dinfo = "+double"
+        elif intype == 4:
+           filepath = self.file4
+           dinfo = "uint"
+           
+        f = open(filepath, 'r')
+        rows = []
+        with f:
+          reader = csv.reader(f, delimiter=',', quotechar='|')
+          for row in reader:
+             rows.append(row)
+        f.close()
+        self.once = len(rows)
+
+        sum = 0
+        for j in range(qtime):
+          offset = j * self.once
+          ts = startts + offset
+          sql = f"insert into {self.database}.{tbname} values"  
+          for i in range(self.once):
+              r = rows[i]
+              sql +=f"({ts + i},'{r[0]}','{r[1]}','{r[2]}','{r[3]}','{r[4]}','{r[5]}','{r[6]}','{r[7]}','{r[8]}','{r[9]}')"
+          
+          t1 = time.time()
+          tdSql.execute(f"{sql};", 1)
+          t2 = time.time() 
+          #print(f"{t2} insert test {j}.")
+          #print(sql)
+          sum += t2 - t1
+
+        sum = sum
+        tbinfo = "10 bigint col/per row"
+        if outtype == 2:
+           tbinfo =  "10 double col/per row"
+        elif outtype == 3:
+           tbinfo =  "10 uint col/per row"
+        print(f" insert {self.once} * {qtime} rows: {sum} s, {dinfo} -> {tbinfo}")
+
+        # tdSql.query(f"select count(*) from {self.database}.{tbname};")
+        # tdSql.checkData(0, 0, once*qtime)
+
+    def run(self):
+        tdSql.prepare(replica = self.replicaVar)
+        # self.make_csv(self.once, 1)
+        # self.make_csv(self.once, 2)
+        # self.make_csv(self.once, 3)
+        # self.make_csv(self.once, 4)
+
+        self.prepare_db()
+        self.test_insert(self.tb1, 1000, self.ts-10000000, 1, 1)
+        self.test_insert(self.tb2, 1000, self.ts-10000000, 2, 2)
+        self.test_insert(self.tb3, 1000, self.ts-10000000, 4, 3)
+        self.test_insert(self.tb2, 1000, self.ts, 1, 2)
+        
+        self.test_insert(self.tb1, 1000, self.ts, 2, 1)
+        self.test_insert(self.tb3, 1000, self.ts, 3, 3)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/system-test/2-query/insert_select.py

@@ -70,7 +70,116 @@ class TDTestCase:
         
         tdSql.error('''insert into %s.tb1 (c8, c9) values(now, 1);'''%(database))
 
-    
+    def use_select_sort(self,database):  
+        ts = 1604298064000
+          
+        tdSql.execute('''drop database if exists %s ;''' %database)
+        tdSql.execute('''create database %s keep 36500 ;'''%(database))
+        tdSql.execute('''use %s;'''%database)
+
+        tdSql.execute('''create stable %s.st (ts timestamp, val int, vt timestamp)  tags (location NCHAR(100));'''%(database))
+        tdSql.execute('''create table %s.t1 using %s.st (location) tags ("0001");'''%(database,database))
+        tdSql.execute('''create table %s.t2 using %s.st (location) tags ("0002");'''%(database,database))
+        tdSql.execute('''create table %s.mt (ts timestamp, val int);'''%(database))
+        
+        
+        tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) ({ts}, 2, {ts});'''%(database))
+        tdSql.query("select ts, val from %s.t1;"%database)
+        tdSql.checkData(0,1,2)
+        
+        ts += 1
+        tdSql.execute(f'''insert into %s.t2 values({ts}, 1, {ts}) ({ts}, 5, {ts}) ({ts}, 2, {ts});'''%(database))
+        tdSql.query("select ts, val from %s.t2;"%database)
+        tdSql.checkData(0,1,2)
+        
+        tdSql.execute('''delete from %s.t2;'''%(database))
+        tdSql.execute('''delete from %s.t1;'''%(database))
+        
+        ts -= 10
+        tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database))
+        ts += 11
+        tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database))
+        ts += 1
+        tdSql.execute(f'''insert into %s.t1 values({ts}, 1, {ts}) %s.t2 values({ts}, 2, {ts});'''%(database,database))
+        
+        tdSql.query("select count(*) from %s.st;"%database)
+        tdSql.checkData(0,0,6)
+        
+        tdSql.query('''select vt, val from %s.st order by vt, val desc;'''%(database))
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(1,1,1)
+        tdSql.checkData(2,1,2)
+        tdSql.checkData(3,1,1)
+        tdSql.checkData(4,1,2)
+        tdSql.checkData(5,1,1)
+        
+        tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val desc;'''%(database,database))
+        tdSql.query("select count(*) from %s.mt;"%database)
+        tdSql.checkData(0,0,3)
+        
+        tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database))
+        tdSql.checkData(0,1,1)
+        tdSql.checkData(1,1,1)
+        tdSql.checkData(2,1,1)
+        
+        tdSql.execute('''delete from %s.mt;'''%(database))
+        tdSql.query('''select vt, val from %s.st order by vt, val asc;'''%(database))
+        tdSql.checkData(0,1,1)
+        tdSql.checkData(1,1,2)
+        tdSql.checkData(2,1,1)
+        tdSql.checkData(3,1,2)
+        tdSql.checkData(4,1,1)
+        tdSql.checkData(5,1,2)
+        
+        tdSql.execute('''insert into %s.mt select vt, val from %s.st order by vt, val asc;'''%(database,database))
+        tdSql.query("select count(*) from %s.mt;"%database)
+        tdSql.checkData(0,0,3)
+        
+        tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database))
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(1,1,2)
+        tdSql.checkData(2,1,2) 
+        
+        tdSql.execute('''delete from %s.mt;'''%(database))
+        tdSql.query('''select vt, val from %s.st order by ts, val asc;'''%(database))
+        tdSql.checkData(0,1,1)
+        tdSql.checkData(1,1,2)
+        tdSql.checkData(2,1,1)
+        tdSql.checkData(3,1,2)
+        tdSql.checkData(4,1,1)
+        tdSql.checkData(5,1,2)
+
+        tdSql.execute('''insert into %s.mt select vt, val from %s.st order by ts, val asc;'''%(database,database))
+        tdSql.query("select count(*) from %s.mt;"%database)
+        tdSql.checkData(0,0,3)
+
+        tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database))
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(1,1,2)
+        tdSql.checkData(2,1,2)
+
+        tdSql.execute('''delete from %s.mt;'''%(database))
+        ts += 1
+        tdSql.execute(f'''insert into %s.t1 values({ts}, -1, {ts}) %s.t2 values({ts}, -2, {ts});'''%(database,database))
+        tdSql.query('''select vt, val from %s.st order by val asc;'''%(database))
+        tdSql.checkData(0,1,-2)
+        tdSql.checkData(1,1,-1)
+        tdSql.checkData(2,1,1)
+        tdSql.checkData(3,1,1)
+        tdSql.checkData(4,1,1)
+        tdSql.checkData(5,1,2)
+        tdSql.checkData(6,1,2)
+        tdSql.checkData(7,1,2)
+
+        tdSql.execute('''insert into %s.mt select vt, val from %s.st order by val asc;'''%(database,database))
+        tdSql.query("select count(*) from %s.mt;"%database)
+        tdSql.checkData(0,0,4)
+        
+        tdSql.query('''select ts, val from %s.mt order by ts asc;'''%(database))
+        tdSql.checkData(0,1,2)
+        tdSql.checkData(1,1,2)
+        tdSql.checkData(2,1,2)
+        tdSql.checkData(3,1,-1)
                             
     def run(self):      
         
@@ -88,6 +197,8 @@ class TDTestCase:
         
         self.users_bug_TD_20592("%s" %self.db) 
         
+        self.use_select_sort("%s" %self.db)
+
         #taos -f sql 
         print("taos -f sql start!")
         taos_cmd1 = "taos -f %s/%s.sql" % (self.testcasePath,self.testcaseFilename)

tests/system-test/2-query/json_tag.py

@@ -394,9 +394,12 @@ class TDTestCase:
         tdSql.execute(f"create table if not exists {dbname}.jsons3(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)")
         tdSql.execute(f"insert into {dbname}.jsons3_1 using {dbname}.jsons3 tags('{{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}}') values(1591060618000, 3, false, 'json3', '你是3')")
         tdSql.execute(f"insert into {dbname}.jsons3_2 using {dbname}.jsons3 tags('{{\"tag1\":5,\"tag2\":\"beijing\"}}') values (1591060638000, 2, true, 'json3', 'sss')")
+        tdSql.execute(f"insert into {dbname}.jsons3_3 using {dbname}.jsons3 tags(NULL) values (1591060638000, 2, true, 'json3', 'sss')")
         tdSql.query(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'")
         tdSql.checkData(0, 0, "sss")
         tdSql.checkData(0, 2, "true")
+        tdSql.query(f"show create table jsons3_3")
+        tdSql.checkNotEqual(tdSql.queryResult[0][1].find("TAGS (null)"), 0)
 
         res = tdSql.getColNameList(f"select 'sss',33,a.jtag->'tag3' from {dbname}.jsons2 a,jsons3 b where a.ts=b.ts and a.jtag->'tag1'=b.jtag->'tag1'")
         cname_list = []

tests/system-test/2-query/last_cache_scan.py

@@ -1,3 +1,4 @@
+from sqlite3 import ProgrammingError
 import taos
 import sys
 import time
@@ -319,10 +320,144 @@ class TDTestCase:
         tdSql.checkData(0, 0, '2018-11-25 19:30:00.000')
         tdSql.checkData(0, 1, '2018-11-25 19:30:01.000')
 
+    def test_cache_scan_with_drop_and_add_column(self):
+        tdSql.query("select last(c10) from meters")
+        tdSql.checkData(0, 0, '2018-11-25 19:30:01')
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"])
+        p.check_returncode()
+        tdSql.query("select last(c10) from meters", queryTimes=1)
+        tdSql.checkData(0, 0, None)
+        tdSql.query('select last(*) from meters', queryTimes=1)
+        tdSql.checkData(0, 10, None)
+        tdSql.query('select last(c10), c10, ts from meters', queryTimes=1)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 2, None)
+
+    def test_cache_scan_with_drop_and_add_column2(self):
+        tdSql.query("select last(c1) from meters")
+        tdSql.checkData(0, 0, '999')
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"])
+        p.check_returncode()
+        tdSql.query("select last(c1) from meters", queryTimes=1)
+        tdSql.checkData(0, 0, None)
+        tdSql.query('select last(*) from meters', queryTimes=1)
+        print(str(tdSql.queryResult))
+        tdSql.checkData(0, 1, None)
+        tdSql.query('select last(c1), c1, ts from meters', queryTimes=1)
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 2, None)
+
+        try:
+            tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1)
+        except Exception as e:
+            if str(e).count('Invalid column name') == 1:
+                print('column has been dropped, the cache has been updated: %s' % (str(e)))
+                return
+            else:
+                raise
+        tdSql.checkRows(1)
+        tdSql.checkCols(5)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(0, 1, None)
+        tdSql.checkData(0, 2, None)
+        tdSql.checkData(0, 3, None)
+        tdSql.checkData(0, 4, None)
+
+        try:
+            tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1)
+        except Exception as e:
+            if str(e).count('Invalid column name') == 1:
+                print('column has been dropped, the cache has been updated: %s' % (str(e)))
+                return
+            else:
+                raise
+        tdSql.checkRows(1)
+        tdSql.checkCols(3)
+        tdSql.checkData(0, 0, None)
+
+    def test_cache_scan_with_drop_column(self):
+        tdSql.query('select last(*) from meters')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(11)
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c9"])
+        p.check_returncode()
+        tdSql.query('select last(*) from meters')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(11)
+        tdSql.checkData(0, 9, None)
+    
+    def test_cache_scan_last_row_with_drop_column(self):
+        tdSql.query('select last_row(*) from meters')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(11)
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c10; alter table test.meters add column c11 int"])
+        p.check_returncode()
+        tdSql.query('select last_row(*) from meters')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(11)
+        tdSql.checkData(0, 10, None)
+        
+    def test_cache_scan_last_row_with_drop_column2(self):
+        tdSql.query('select last_row(c1) from meters')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"])
+        p.check_returncode()
+        tdSql.query('select last_row(c1) from meters', queryTimes=1)
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        tdSql.checkData(0, 0, None)
+
+    def test_cache_scan_last_row_with_partition_by(self):
+        tdSql.query('select last(c1) from meters partition by t1')
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        tdSql.checkRows(5)
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"])
+        p.check_returncode()
+        tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1)
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        tdSql.checkRows(5)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(1, 0, None)
+        tdSql.checkData(2, 0, None)
+        tdSql.checkData(3, 0, None)
+        tdSql.checkData(4, 0, None)
+
+    def test_cache_scan_last_row_with_partition_by_tbname(self):
+        tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1)
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        tdSql.checkRows(10)
+        p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"])
+        p.check_returncode()
+        tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1)
+        print(str(tdSql.queryResult))
+        tdSql.checkCols(1)
+        tdSql.checkRows(10)
+        tdSql.checkData(0, 0, None)
+        tdSql.checkData(1, 0, None)
+        tdSql.checkData(2, 0, None)
+        tdSql.checkData(3, 0, None)
+        tdSql.checkData(4, 0, None)
+
+
+
     def run(self):
         self.prepareTestEnv()
         #time.sleep(99999999)
         self.test_last_cache_scan()
+        #self.test_cache_scan_with_drop_and_add_column()
+        self.test_cache_scan_with_drop_and_add_column2()
+        #self.test_cache_scan_with_drop_column()
+        #self.test_cache_scan_last_row_with_drop_column()
+        #self.test_cache_scan_last_row_with_drop_column2()
+        #self.test_cache_scan_last_row_with_partition_by()
+        #self.test_cache_scan_last_row_with_partition_by_tbname()
 
     def stop(self):
         tdSql.close()