Merge branch '3.0' into fix/snodeBackendCrash

.gitignore

@@ -11,6 +11,7 @@ CMakeSettings.json
 cmake-build-debug/
 cmake-build-release/
 cscope.out
+cscope.files
 .DS_Store
 debug/
 release/

CMakeLists.txt

@@ -41,6 +41,7 @@ add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(utils)
 add_subdirectory(examples/c)
+add_subdirectory(tests)
 include(${TD_SUPPORT_DIR}/cmake.install)
 
 # docs 

cmake/cmake.define

@@ -181,17 +181,17 @@ ELSE ()
         ENDIF()
         MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
 
-        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
-            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
-        ENDIF()
-
-        IF (COMPILER_SUPPORT_AVX512VL)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
-        MESSAGE(STATUS "avx512vl supported by compiler")
-        ENDIF()
+#        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
+#            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
+#        ENDIF()
+#
+#        IF (COMPILER_SUPPORT_AVX512VL)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
+#        MESSAGE(STATUS "avx512vl supported by compiler")
+#        ENDIF()
     ENDIF()
 
     # build mode

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -262,6 +262,63 @@ The following types may be returned:
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
+
+Prepare data
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+Execute query
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+Return results
+
+```json
+{
+     "code": 0,
+     "column_meta": [
+         [
+             "ts",
+             "TIMESTAMP",
+             8
+         ],
+         [
+             "c1",
+             "VARBINARY",
+             20
+         ],
+         [
+             "c2",
+             "GEOMETRY",
+             100
+         ]
+     ],
+     "data": [
+         [
+             "2023-11-01T06:28:15.210Z",
+             "7f8290",
+             "010100000000000000000059400000000000005940"
+         ]
+     ],
+     "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
 
 #### Errors
 

docs/examples/python/bind_param_example.py

@@ -20,7 +20,7 @@ def get_ts(ts: str):
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/conn_websocket_pandas.py

@@ -4,7 +4,7 @@ import taos
 
 taos_conn = taos.connect()
 taos_conn.execute('drop database if exists power')
-taos_conn.execute('create database if not exists power wal_retention_period 3600')
+taos_conn.execute('create database if not exists power wal_retention_period 3600 keep 36500 ')
 taos_conn.execute("use power")
 taos_conn.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")

docs/examples/python/connect_rest_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power")
-cursor.execute("CREATE DATABASE power")
+cursor.execute("CREATE DATABASE power keep 36500 ")
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)")
 

docs/examples/python/connect_rest_with_req_id_examples.py

@@ -11,7 +11,7 @@ conn = connect(url="http://localhost:6041",
 # create STable
 cursor = conn.cursor()
 cursor.execute("DROP DATABASE IF EXISTS power", req_id=1)
-cursor.execute("CREATE DATABASE power", req_id=2)
+cursor.execute("CREATE DATABASE power keep 36500", req_id=2)
 cursor.execute(
     "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)", req_id=3)
 

docs/examples/python/connect_websocket_examples.py

@@ -6,13 +6,13 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy")
-conn.execute("create database if not exists connwspy wal_retention_period 3600")
+conn.execute("create database if not exists connwspy wal_retention_period 3600 keep 36500 ")
 conn.execute("use connwspy")
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)")
 conn.execute("create table if not exists tb1 using stb tags (1)")
 conn.execute("insert into tb1 values (now, 1)")
-conn.execute("insert into tb1 values (now, 2)")
-conn.execute("insert into tb1 values (now, 3)")
+conn.execute("insert into tb1 values (now+1s, 2)")
+conn.execute("insert into tb1 values (now+2s, 3)")
 
 r = conn.execute("select * from stb")
 result = conn.query("select * from stb")

docs/examples/python/connect_websocket_with_req_id_examples.py

@@ -6,7 +6,7 @@ conn = taosws.connect("taosws://root:taosdata@localhost:6041")
 
 # ANCHOR: basic
 conn.execute("drop database if exists connwspy", req_id=1)
-conn.execute("create database if not exists connwspy", req_id=2)
+conn.execute("create database if not exists connwspy keep 36500", req_id=2)
 conn.execute("use connwspy", req_id=3)
 conn.execute("create table if not exists stb (ts timestamp, c1 int) tags (t1 int)", req_id=4)
 conn.execute("create table if not exists tb1 using stb tags (1)", req_id=5)

docs/examples/python/connection_usage_native_reference.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")

docs/examples/python/connection_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ import taos
 conn = taos.connect()
 # Execute a sql, ignore the result set, just get affected rows. It's useful for DDL and DML statement.
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 # change database. same as execute "USE db"
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)

docs/examples/python/cursor_usage_native_reference.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test")
-cursor.execute("CREATE DATABASE test")
+cursor.execute("CREATE DATABASE test keep 36500")
 cursor.execute("USE test")
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 

docs/examples/python/cursor_usage_native_reference_with_req_id.py

@@ -4,7 +4,7 @@ conn = taos.connect()
 cursor = conn.cursor()
 
 cursor.execute("DROP DATABASE IF EXISTS test", req_id=1)
-cursor.execute("CREATE DATABASE test", req_id=2)
+cursor.execute("CREATE DATABASE test keep 36500", req_id=2)
 cursor.execute("USE test", req_id=3)
 cursor.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=4)
 

docs/examples/python/fast_write_example.py

@@ -160,7 +160,7 @@ def main(infinity):
 
     conn = get_connection()
     conn.execute("DROP DATABASE IF EXISTS test")
-    conn.execute("CREATE DATABASE IF NOT EXISTS test")
+    conn.execute("CREATE DATABASE IF NOT EXISTS test keep 36500")
     conn.execute("CREATE STABLE IF NOT EXISTS test.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")
     conn.close()

docs/examples/python/json_protocol_example.py

@@ -16,7 +16,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/kafka_example_common.py

@@ -5,7 +5,7 @@ LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanD
              'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale',
              'California.SantaClara', 'California.Cupertino']
 
-CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
+CREATE_DATABASE_SQL = 'create database if not exists {} keep 36500 duration 10 buffer 16 wal_level 1 wal_retention_period 3600'
 USE_DATABASE_SQL = 'use {}'
 DROP_TABLE_SQL = 'drop table if exists meters'
 DROP_DATABASE_SQL = 'drop database if exists {}'

docs/examples/python/line_protocol_example.py

@@ -15,7 +15,7 @@ def get_connection():
 
 def create_database(conn):
     # the default precision is ms (microsecond), but we use us(microsecond) here.
-    conn.execute("CREATE DATABASE test precision 'us'")
+    conn.execute("CREATE DATABASE test precision 'us' keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/multi_bind_example.py

@@ -71,7 +71,7 @@ def insert_data():
 def create_stable():
     conn = taos.connect()
     try:
-        conn.execute("CREATE DATABASE power")
+        conn.execute("CREATE DATABASE power keep 36500")
         conn.execute("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                      "TAGS (location BINARY(64), groupId INT)")
     finally:

docs/examples/python/native_insert_example.py

@@ -18,7 +18,7 @@ def get_connection() -> taos.TaosConnection:
 
 
 def create_stable(conn: taos.TaosConnection):
-    conn.execute("CREATE DATABASE power")
+    conn.execute("CREATE DATABASE power keep 36500")
     conn.execute("USE power")
     conn.execute("CREATE STABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) "
                  "TAGS (location BINARY(64), groupId INT)")

docs/examples/python/result_set_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test")
-conn.execute("CREATE DATABASE test")
+conn.execute("CREATE DATABASE test keep 36500")
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)")
 # prepare data

docs/examples/python/result_set_with_req_id_examples.py

@@ -2,7 +2,7 @@ import taos
 
 conn = taos.connect()
 conn.execute("DROP DATABASE IF EXISTS test", req_id=1)
-conn.execute("CREATE DATABASE test", req_id=2)
+conn.execute("CREATE DATABASE test keep 36500", req_id=2)
 conn.select_db("test")
 conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)", req_id=3)
 # prepare data

docs/examples/python/schemaless_insert.py

@@ -3,7 +3,7 @@ import taos
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_raw.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_req_id.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_raw_ttl.py

@@ -10,9 +10,9 @@ try:
     conn.execute("drop database if exists %s" % dbname)
 
     if taos.IS_V3:
-        conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s schemaless 1 precision 'ns' keep 36500" % dbname)
     else:
-        conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname)
+        conn.execute("create database if not exists %s update 2 precision 'ns' keep 36500" % dbname)
 
     conn.select_db(dbname)
 

docs/examples/python/schemaless_insert_req_id.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/schemaless_insert_ttl.py

@@ -4,7 +4,7 @@ from taos import SmlProtocol, SmlPrecision
 conn = taos.connect()
 dbname = "pytest_line"
 conn.execute("drop database if exists %s" % dbname)
-conn.execute("create database if not exists %s precision 'us'" % dbname)
+conn.execute("create database if not exists %s precision 'us' keep 36500" % dbname)
 conn.select_db(dbname)
 
 lines = [

docs/examples/python/sql_writer.py

@@ -10,7 +10,7 @@ class SQLWriter:
         self._tb_tags = {}
         self._conn = get_connection_func()
         self._max_sql_length = self.get_max_sql_length()
-        self._conn.execute("create database if not exists test")
+        self._conn.execute("create database if not exists test keep 36500")
         self._conn.execute("USE test")
 
     def get_max_sql_length(self):

docs/examples/python/stmt_example.py

@@ -10,7 +10,7 @@ db_name = 'test_ws_stmt'
 def before():
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/stmt_websocket_example.py

@@ -9,7 +9,7 @@ import taos
 def before_test(db_name):
     taos_conn = taos.connect()
     taos_conn.execute("drop database if exists %s" % db_name)
-    taos_conn.execute("create database %s" % db_name)
+    taos_conn.execute("create database %s keep 36500" % db_name)
     taos_conn.select_db(db_name)
     taos_conn.execute("create table t1 (ts timestamp, a int, b float, c varchar(10))")
     taos_conn.execute(

docs/examples/python/telnet_line_protocol_example.py

@@ -19,7 +19,7 @@ def get_connection():
 
 
 def create_database(conn):
-    conn.execute("CREATE DATABASE test")
+    conn.execute("CREATE DATABASE test keep 36500")
     conn.execute("USE test")
 
 

docs/examples/python/tmq_assignment_example.py

@@ -7,7 +7,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

docs/examples/python/tmq_example.py

@@ -6,7 +6,7 @@ def init_tmq_env(db, topic):
     conn = taos.connect()
     conn.execute("drop topic if exists {}".format(topic))
     conn.execute("drop database if exists {}".format(db))
-    conn.execute("create database if not exists {} wal_retention_period 3600".format(db))
+    conn.execute("create database if not exists {} wal_retention_period 3600 keep 36500".format(db))
     conn.select_db(db)
     conn.execute(
         "create stable if not exists stb1 (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))")

docs/examples/python/tmq_websocket_assgnment_example.py

@@ -6,7 +6,7 @@ def prepare():
     conn = taos.connect()
     conn.execute("drop topic if exists tmq_assignment_demo_topic")
     conn.execute("drop database if exists tmq_assignment_demo_db")
-    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600")
+    conn.execute("create database if not exists tmq_assignment_demo_db wal_retention_period 3600 keep 36500")
     conn.select_db("tmq_assignment_demo_db")
     conn.execute(
         "create table if not exists tmq_assignment_demo_table (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)")

docs/zh/08-connector/02-rest-api.mdx

@@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
+
+准备数据
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+执行查询
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+返回结果
+
+```json
+{
+    "code": 0,
+    "column_meta": [
+        [
+            "ts",
+            "TIMESTAMP",
+            8
+        ],
+        [
+            "c1",
+            "VARBINARY",
+            20
+        ],
+        [
+            "c2",
+            "GEOMETRY",
+            100
+        ]
+    ],
+    "data": [
+        [
+            "2023-11-01T06:28:15.210Z",
+            "7f8290",
+            "010100000000000000000059400000000000005940"
+        ]
+    ],
+    "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)  格式
 
 #### 错误
 

include/common/cos.h

@@ -34,7 +34,7 @@ extern int32_t tsS3UploadDelaySec;
 int32_t s3Init();
 void    s3CleanUp();
 int32_t s3PutObjectFromFile(const char *file, const char *object);
-int32_t s3PutObjectFromFile2(const char *file, const char *object);
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
 void    s3DeleteObjectsByPrefix(const char *prefix);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);

include/common/cos_cp.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_COS_CP_H_
+#define _TD_COMMON_COS_CP_H_
+
+#include "os.h"
+#include "tdef.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+  COS_CP_TYPE_UPLOAD,   // upload
+  COS_CP_TYPE_DOWNLOAD  // download
+} ECpType;
+
+typedef struct {
+  int32_t  index;      // the index of part, start from 0
+  int64_t  offset;     // the offset point of part
+  int64_t  size;       // the size of part
+  int      completed;  // COS_TRUE completed, COS_FALSE uncompleted
+  char     etag[128];  // the etag of part, for upload
+  uint64_t crc64;
+} SCheckpointPart;
+
+typedef struct {
+  ECpType   cp_type;  // 0 upload, 1 download
+  char      md5[64];  // the md5 of checkout content
+  TdFilePtr thefile;  // the handle of checkpoint file
+
+  char    file_path[TSDB_FILENAME_LEN];  // local file path
+  int64_t file_size;                     // local file size, for upload
+  int32_t file_last_modified;            // local file last modified time, for upload
+  char    file_md5[64];                  // md5 of the local file content, for upload, reserved
+
+  char    object_name[128];          // object name
+  int64_t object_size;               // object size, for download
+  char    object_last_modified[64];  // object last modified time, for download
+  char    object_etag[128];          // object etag, for download
+
+  char upload_id[128];  // upload id
+
+  int              part_num;   // the total number of parts
+  int64_t          part_size;  // the part size, byte
+  SCheckpointPart* parts;      // the parts of local or object, from 0
+} SCheckpoint;
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
+void    cos_cp_close(TdFilePtr fd);
+void    cos_cp_remove(char const* filepath);
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
+int32_t cos_cp_dump(SCheckpoint* checkpoint);
+void    cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
+void    cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
+void    cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                            char const* upload_id, int64_t part_size);
+bool    cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
+
+void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
+                           char const* object_lmtime, char const* object_etag, int64_t part_size);
+bool cos_cp_is_valid_download(SCheckpoint* checkpoint, char const* object_name, int64_t object_size,
+                              char const* object_lmtime, char const* object_etag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_COS_CP_H_*/

include/common/tcommon.h

@@ -253,6 +253,7 @@ typedef struct SQueryTableDataCond {
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
+  bool         notLoadData;    // response the actual data, not only the rows in the attribute of info.row of ssdatablock
 } SQueryTableDataCond;
 
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);

include/common/tglobal.h

@@ -107,8 +107,9 @@ extern int32_t  tsMonitorMaxLogs;
 extern bool     tsMonitorComp;
 
 // audit
-extern bool tsEnableAudit;
-extern bool tsEnableAuditCreateTable;
+extern bool     tsEnableAudit;
+extern bool     tsEnableAuditCreateTable;
+extern int32_t  tsAuditInterval;
 
 // telem
 extern bool     tsEnableTelem;

include/common/tmsg.h

@@ -295,7 +295,36 @@ typedef enum ENodeType {
   QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_GRANT_STMT,
   QUERY_NODE_REVOKE_STMT,
-  QUERY_NODE_SHOW_DNODES_STMT,
+  // placeholder for [152, 180]
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
+  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_DELETE_STMT,
+  QUERY_NODE_INSERT_STMT,
+  QUERY_NODE_QUERY,
+  QUERY_NODE_SHOW_DB_ALIVE_STMT,
+  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_RESTORE_DNODE_STMT,
+  QUERY_NODE_RESTORE_QNODE_STMT,
+  QUERY_NODE_RESTORE_MNODE_STMT,
+  QUERY_NODE_RESTORE_VNODE_STMT,
+  QUERY_NODE_PAUSE_STREAM_STMT,
+  QUERY_NODE_RESUME_STREAM_STMT,
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
+
+  // show statement nodes
+  // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
+  QUERY_NODE_SHOW_DNODES_STMT = 400,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
@@ -324,31 +353,6 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_VIEWS_STMT,
-  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
-  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
-  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
-  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
-  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
-  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
-  QUERY_NODE_SHOW_SCORES_STMT,
-  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
-  QUERY_NODE_KILL_CONNECTION_STMT,
-  QUERY_NODE_KILL_QUERY_STMT,
-  QUERY_NODE_KILL_TRANSACTION_STMT,
-  QUERY_NODE_DELETE_STMT,
-  QUERY_NODE_INSERT_STMT,
-  QUERY_NODE_QUERY,
-  QUERY_NODE_SHOW_DB_ALIVE_STMT,
-  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
-  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
-  QUERY_NODE_RESTORE_DNODE_STMT,
-  QUERY_NODE_RESTORE_QNODE_STMT,
-  QUERY_NODE_RESTORE_MNODE_STMT,
-  QUERY_NODE_RESTORE_VNODE_STMT,
-  QUERY_NODE_PAUSE_STREAM_STMT,
-  QUERY_NODE_RESUME_STREAM_STMT,
-  QUERY_NODE_CREATE_VIEW_STMT,
-  QUERY_NODE_DROP_VIEW_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,

include/common/tmsgdef.h

@@ -26,10 +26,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) "null",
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) MSG, MSG "-rsp",
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   char *tMsgInfo[] = {
 
@@ -37,20 +37,20 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) 
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
   int32_t tMsgRangeDict[] = {
 
 #elif defined(TD_MSG_NUMBER_)
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE##_NUM, TYPE##_RSP_NUM,
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -58,10 +58,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(type)
+  #define TD_CLOSE_MSG_SEG(type)
 
   int32_t tMsgDict[] = {
 
@@ -70,10 +70,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_SEG_CODE,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_TYPE(TYPE)
+  #define TD_CLOSE_MSG_SEG(TYPE)
 
   enum {
 
@@ -82,10 +82,10 @@
 
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
-  #undef TD_CLOSE_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
-  #define TD_CLOSE_MSG_TYPE(TYPE) TYPE,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
 
   enum { // WARN: new msg should be appended to segment tail
 #endif
@@ -109,7 +109,7 @@
   TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_DND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
@@ -218,7 +218,7 @@
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
@@ -268,7 +268,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
@@ -283,7 +283,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_TASK_NOTIFY, "task-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SCH_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
@@ -301,11 +301,11 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_HTASK_DROP, "stream-htask-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_MON_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
@@ -337,7 +337,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_SYNC_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG) 
 
 
   TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
@@ -348,7 +348,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_VND_STREAM_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
@@ -362,10 +362,10 @@ TD_DEF_MSG_TYPE(TDMT_VND_ALTER_CONFIG, "alter-config", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_COMMITTEDINFO, "vnode-tmq-committedinfo", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL)
-  TD_CLOSE_MSG_TYPE(TDMT_END_TMQ_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_TMQ_MSG) 
 
   TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
-  
+  TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG) 
 
   
 

include/common/ttypes.h

@@ -275,9 +275,11 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT64(_t)     ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
+#define IS_VALID_UINT64(_t)    ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -37,6 +37,10 @@ typedef struct SVariant {
   };
 } SVariant;
 
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value);
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value);
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 

include/libs/audit/audit.h

@@ -23,13 +23,13 @@
 #include "tjson.h"
 #include "tmsgcb.h"
 #include "trpc.h"
-#include "mnode.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #define AUDIT_DETAIL_MAX 65472
+#define AUDIT_OPERATION_LEN 20
 
 typedef struct {
   const char *server;
@@ -37,13 +37,28 @@ typedef struct {
   bool        comp;
 } SAuditCfg;
 
+typedef struct {
+  int64_t curTime;
+  char    strClusterId[TSDB_CLUSTER_ID_LEN];
+  char    clientAddress[50];
+  char    user[TSDB_USER_LEN];
+  char    operation[AUDIT_OPERATION_LEN];
+  char    target1[TSDB_DB_NAME_LEN]; //put db name
+  char    target2[TSDB_STREAM_NAME_LEN]; //put stb name, table name, topic name, user name, stream name, use max
+  char*   detail;
+} SAuditRecord;
+
 int32_t auditInit(const SAuditCfg *pCfg);
+void    auditCleanup();
 void    auditSend(SJson *pJson);
 void    auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, 
                     char *detail, int32_t len);
+void    auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, 
+                    char *detail, int32_t len);
+void    auditSendRecordsInBatch();
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /*_TD_MONITOR_H_*/
+#endif /*_TD_AUDIT_H_*/

include/libs/executor/storageapi.h

@@ -152,10 +152,21 @@ typedef struct {
 
 // clang-format off
 /*-------------------------------------------------new api format---------------------------------------------------*/
+typedef enum {
+  TSD_READER_NOTIFY_DURATION_START
+} ETsdReaderNotifyType;
+
+typedef union {
+  struct {
+    int32_t filesetId;
+  } duration;
+} STsdReaderNotifyInfo;
+
+typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param);
+
 typedef struct TsdReader {
   int32_t      (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
-                           SHashObj** pIgnoreTables);
+                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
   void         (*tsdReaderClose)();
   void         (*tsdSetReaderTaskId)(void *pReader, const char *pId);
   int32_t      (*tsdSetQueryTableList)();
@@ -170,6 +181,9 @@ typedef struct TsdReader {
   int32_t      (*tsdReaderGetDataBlockDistInfo)();
   int64_t      (*tsdReaderGetNumOfInMemRows)();
   void         (*tsdReaderNotifyClosing)();
+
+  void         (*tsdSetFilesetDelimited)(void* pReader);
+  void         (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 } TsdReader;
 
 typedef struct SStoreCacheReader {

include/libs/nodes/plannodes.h

@@ -118,6 +118,7 @@ typedef struct SScanLogicNode {
   bool          igLastNull;
   bool          groupOrderScan;
   bool          onlyMetaCtbIdx; // for tag scan with no tbname
+  bool          filesetDelimited; // returned blocks delimited by fileset
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -432,6 +433,7 @@ typedef struct STableScanPhysiNode {
   int8_t         igExpired;
   bool           assignBlockUid;
   int8_t         igCheckUpdate;
+  bool           filesetDelimited;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;

include/libs/stream/tstream.h

@@ -509,11 +509,8 @@ typedef struct SStreamMeta {
   SArray*  chkpSaved;
   SArray*  chkpInUse;
   SRWLatch chkpDirLock;
-
-  void*   qHandle;
-  int32_t pauseTaskNum;
-
-  void* bkdChkptMgt;
+  void*    qHandle;
+  void*    bkdChkptMgt;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -840,11 +837,10 @@ int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int3
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
-int32_t      streamMetaReopen(SStreamMeta* pMeta);
+void         streamMetaClear(SStreamMeta* pMeta);
 void         streamMetaInitBackend(SStreamMeta* pMeta);
 int32_t      streamMetaCommit(SStreamMeta* pMeta);
 int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
-int32_t      streamMetaReloadAllTasks(SStreamMeta* pMeta);
 int64_t      streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
 int32_t      streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);

include/libs/sync/sync.h

@@ -33,11 +33,12 @@ extern "C" {
 #define SYNC_MAX_PROGRESS_WAIT_MS    4000
 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
 #define SYNC_MAX_RECV_TIME_RANGE_MS  1200
-#define SYNC_DEL_WAL_MS              (1000 * 60)
 #define SYNC_ADD_QUORUM_COUNT        3
 #define SYNC_VNODE_LOG_RETENTION     (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
 #define SNAPSHOT_WAIT_MS             1000 * 5
 
+#define SYNC_WAL_LOG_RETENTION_SIZE (8LL * 1024 * 1024 * 1024)
+
 #define SYNC_MAX_RETRY_BACKOFF         5
 #define SYNC_LOG_REPL_RETRY_WAIT_MS    100
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
@@ -219,6 +220,7 @@ typedef struct SSyncLogStore {
 
   SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
   SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogIndexRetention)(struct SSyncLogStore* pLogStore, int64_t bytes);
   SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
 
   int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);
@@ -289,6 +291,9 @@ const char* syncStr(ESyncState state);
 
 int32_t    syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
 
+// util
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/wal/wal.h

@@ -225,6 +225,7 @@ bool walIsEmpty(SWal *);
 int64_t walGetFirstVer(SWal *);
 int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);
+int64_t walGetVerRetention(SWal *pWal, int64_t bytes);
 int64_t walGetCommittedVer(SWal *);
 int64_t walGetAppliedVer(SWal *);
 

include/os/osString.h

@@ -47,18 +47,13 @@ typedef int32_t TdUcs4;
 #define strtof      STR_TO_F_FUNC_TAOS_FORBID
 #endif
 
-#ifdef WINDOWS
-#define tstrdup(str) _strdup(str)
-#else
-#define tstrdup(str) strdup(str)
-#endif
-
 #define tstrncpy(dst, src, size)   \
   do {                             \
     strncpy((dst), (src), (size)); \
     (dst)[(size)-1] = 0;           \
   } while (0)
 
+char   *tstrdup(const char *src);
 int32_t taosUcs4len(TdUcs4 *ucs4);
 int64_t taosStr2int64(const char *str);
 

include/util/tdef.h

@@ -288,6 +288,7 @@ typedef enum ELogicConditionType {
 #define TSDB_CONN_ACTIVE_KEY_LEN 255
 
 #define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024)
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120  // default payload size, greater than PATH_MAX value

source/client/inc/clientInt.h

@@ -205,7 +205,7 @@ typedef struct SRequestSendRecvBody {
   __taos_async_fn_t queryFp;
   __taos_async_fn_t fetchFp;
   EQueryExecMode    execMode;
-  void*             param;
+  void*             interParam;
   SDataBuf          requestMsg;
   int64_t           queryJob;  // query job, created according to sql query DAG.
   int32_t           subplanNum;
@@ -287,6 +287,7 @@ typedef struct SRequestObj {
 typedef struct SSyncQueryParam {
   tsem_t       sem;
   SRequestObj* pRequest;
+  void*        userParam;
 } SSyncQueryParam;
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
@@ -420,6 +421,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
 int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
 void    returnToUser(SRequestObj* pRequest);
 void    stopAllQueries(SRequestObj *pRequest);
+void    doRequestCallback(SRequestObj* pRequest, int32_t code);
 void    freeQueryParam(SSyncQueryParam* param);
 
 #ifdef TD_ENTERPRISE

source/client/inc/clientSml.h

@@ -65,14 +65,32 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-//#define TS        "_ts"
-//#define TS_LEN    3
 #define VALUE     "_value"
-#define VALUE_LEN 6
+#define VALUE_LEN (sizeof(VALUE)-1)
 
 #define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 10
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
+
+#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+
+#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
+
+#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
+
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
 
 typedef enum {
   SCHEMA_ACTION_NULL,
@@ -83,18 +101,6 @@ typedef enum {
   SCHEMA_ACTION_CHANGE_TAG_SIZE,
 } ESchemaAction;
 
-typedef struct {
-  const void  *key;
-  int32_t      keyLen;
-  void        *value;
-  bool         used;
-}Node;
-
-typedef struct NodeList{
-  Node             data;
-  struct NodeList* next;
-}NodeList;
-
 typedef struct {
   char *measure;
   char *tags;
@@ -117,7 +123,6 @@ typedef struct {
   int32_t     sTableNameLen;
   char        childTableName[TSDB_TABLE_NAME_LEN];
   uint64_t    uid;
-//  void       *key;        // for openTsdb
 
   SArray *tags;
 
@@ -161,7 +166,8 @@ typedef struct {
 typedef struct {
   int64_t id;
 
-  SMLProtocolType protocol;
+  TSDB_SML_PROTOCOL_TYPE protocol;
+
   int8_t          precision;
   bool            reRun;
   bool            dataFormat;  // true means that the name and order of keys in each line are the same(only for influx protocol)
@@ -201,29 +207,8 @@ typedef struct {
   bool         needModifySchema;
 } SSmlHandle;
 
-#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
-
-#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
-
-#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
-
-#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
-
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
-
-#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
-  for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
-      MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      keyLen--;                              \
-    }                                        \
-  }
-
-extern int64_t smlFactorNS[3];
-extern int64_t smlFactorS[3];
+extern int64_t smlFactorNS[];
+extern int64_t smlFactorS[];
 
 typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
@@ -231,16 +216,10 @@ SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
 int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-//int           nodeListSize(NodeList* list);
-bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
 int64_t       smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
-int8_t        smlGetTsTypeByLen(int32_t len);
 SSmlTableInfo*    smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
 SSmlSTableMeta*   smlBuildSTableMeta(bool isDataFormat);
 int32_t           smlSetCTableName(SSmlTableInfo *oneTable);
@@ -253,12 +232,45 @@ int32_t           smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
 uint8_t           smlGetTimestampLen(int64_t num);
 void              smlDestroyTableInfo(void *para);
 
-void freeSSmlKv(void* data);
+void    freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
-void    smlStrReplace(char* src, int32_t len);
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
+bool            isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+bool            isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+int32_t         smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlJoinMeasureTag(SSmlLineInfo *elements);
+void            smlBuildTsKv(SSmlKv *kv, int64_t ts);
+int32_t         smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
+int32_t         smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
+
+static inline bool smlDoubleToInt64OverFlow(double num) {
+  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
+  return false;
+}
+
+static inline void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
+static inline int8_t smlGetTsTypeByLen(int32_t len) {
+  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
+    return TSDB_TIME_PRECISION_SECONDS;
+  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+    return TSDB_TIME_PRECISION_MILLI;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -316,6 +316,15 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
+  SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (interParam == NULL) {
+    doDestroyRequest(pRequest);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  tsem_init(&interParam->sem, 0, 0);
+  interParam->pRequest = pRequest;
+  pRequest->body.interParam = interParam;
 
   pRequest->resType = RES_TYPE__QUERY;
   pRequest->requestId = reqid == 0 ? generateRequestId() : reqid;
@@ -437,12 +446,10 @@ void doDestroyRequest(void *p) {
     deregisterRequest(pRequest);
   }
 
-  if (pRequest->syncQuery) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
+  if (pRequest->body.interParam) {
+    tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
   }
+  taosMemoryFree(pRequest->body.interParam);
 
   qDestroyQuery(pRequest->pQuery);
   nodesDestroyAllocator(pRequest->allocatorRefId);
@@ -732,7 +739,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         return -1;
       }
       newstr[0] = '"';
-      strncpy(newstr+1, str, len);
+      memcpy(newstr+1, str, len);
       newstr[len + 1] = '"';
       newstr[len + 2] = '\0';
       str = newstr;

source/client/src/clientImpl.c

@@ -196,21 +196,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   (*pRequest)->sqlLen = sqlLen;
   (*pRequest)->validateOnly = validateSql;
 
-  SSyncQueryParam* newpParam = NULL;
-  if (param == NULL) {
-    newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-    if (newpParam == NULL) {
-      destroyRequest(*pRequest);
-      *pRequest = NULL;
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    tsem_init(&newpParam->sem, 0, 0);
-    newpParam->pRequest = (*pRequest);
-    param = newpParam;
-  }
-
-  (*pRequest)->body.param = param;
+  ((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
 
   STscObj* pTscObj = (*pRequest)->pTscObj;
   int32_t  err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
@@ -218,7 +204,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   if (err) {
     tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
              (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-    freeQueryParam(newpParam);
     destroyRequest(*pRequest);
     *pRequest = NULL;
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -230,7 +215,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
         nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
       tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
                (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-      freeQueryParam(newpParam);
       destroyRequest(*pRequest);
       *pRequest = NULL;
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -336,7 +320,7 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb
 void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   SRetrieveTableRsp* pRsp = NULL;
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -358,18 +342,18 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
              pRequest->requestId);
   }
 
-  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+  doRequestCallback(pRequest, code);
 }
 
 int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
   // drop table if exists not_exists_table
   if (NULL == pQuery->pCmdMsg) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -384,7 +368,7 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   int64_t transporterId = 0;
   int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
   if (code) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
   return code;
 }
@@ -913,7 +897,7 @@ void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
 void returnToUser(SRequestObj* pRequest) {
   if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) {
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
     return;
   }
 
@@ -921,7 +905,7 @@ void returnToUser(SRequestObj* pRequest) {
   if (pUserReq) {
     pUserReq->code = pRequest->code;
     // return to client
-    pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code);
+    doRequestCallback(pUserReq, pUserReq->code);
     releaseRequest(pRequest->relation.userRefId);
     return;
   } else {
@@ -1031,7 +1015,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
     pRequest->pWrapper = NULL;
 
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1186,7 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
       pRequest->code = terrno;
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 
   // todo not to be released here
@@ -1199,7 +1183,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
   int32_t code = 0;
 
   if (pRequest->parseOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -1233,11 +1217,11 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
       pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
-      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+      doRequestCallback(pRequest, 0);
       break;
     default:
       tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
-      pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
+      doRequestCallback(pRequest, -1);
       break;
   }
 }
@@ -1703,8 +1687,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
 }
 
 static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
-  SSyncQueryParam* pParam = param;
-  tsem_post(&pParam->sem);
+  tsem_t* sem = param;
+  tsem_post(sem);
 }
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
@@ -1722,10 +1706,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
 
     // convert ucs4 to native multi-bytes string
     pResultInfo->convertUcs4 = convertUcs4;
-
-    SSyncQueryParam* pParam = pRequest->body.param;
-    taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
-    tsem_wait(&pParam->sem);
+    tsem_t sem;
+    tsem_init(&sem, 0, 0);
+    taos_fetch_rows_a(pRequest, syncFetchFn, &sem);
+    tsem_wait(&sem);
+    tsem_destroy(&sem);
   }
 
   if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@@ -2492,6 +2477,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   tscDebug("taos_query start with sql:%s", sql);
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (NULL == param) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
@@ -2501,9 +2490,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
+  taosMemoryFree(param);
 
   tscDebug("taos_query end with sql:%s", sql);
 
@@ -2517,19 +2505,20 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
   }
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (param == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
   tsem_wait(&param->sem);
-
   SRequestObj* pRequest = NULL;
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
-
+  taosMemoryFree(param);
   return pRequest;
 }
 
@@ -2547,13 +2536,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     pRequest->code = code;
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
@@ -2574,20 +2563,12 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
     atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
   }
 
-  pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
+  pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, pResultInfo->numOfRows);
 }
 
 void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
-  if (pRequest->syncQuery && pRequest->body.param != param) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
-    pRequest->syncQuery = false;
-  }
-  
   pRequest->body.fetchFp = fp;
-  pRequest->body.param = param;
+  ((SSyncQueryParam *)pRequest->body.interParam)->userParam = param;
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
 
@@ -2625,6 +2606,10 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
   schedulerFetchRows(pRequest->body.queryJob, &req);
 }
 
+void doRequestCallback(SRequestObj* pRequest, int32_t code) {
+  pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
+}
+
 int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
 #ifndef TD_ENTERPRISE
   return TSDB_CODE_SUCCESS;
@@ -2633,4 +2618,3 @@ int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool pa
 #endif
 }
 
-

source/client/src/clientMain.c

@@ -1095,7 +1095,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1112,7 +1112,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1210,7 +1210,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
     terrno = code;
     pRequest->code = code;
     tscDebug("call sync query cb with code: %s", tstrerror(code));
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
     return;
   }
 
@@ -1242,7 +1242,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1545,12 +1545,12 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
 
   conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
-  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.param, NULL);
+  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.interParam, NULL);
   if (code) {
     goto _return;
   }
 
-  SSyncQueryParam *pParam = pRequest->body.param;
+  SSyncQueryParam *pParam = pRequest->body.interParam;
   tsem_wait(&pParam->sem);
 
 _return:

source/client/src/clientMsgHandler.c

@@ -41,7 +41,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -199,7 +199,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (pRequest->body.queryFp) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -235,7 +235,8 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     setErrno(pRequest, code);
 
     if (pRequest->body.queryFp != NULL) {
-      pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+      doRequestCallback(pRequest, pRequest->code);
+
     } else {
       tsem_post(&pRequest->body.rspSem);
     }
@@ -299,7 +300,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -343,7 +344,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -380,7 +381,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -420,7 +421,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -534,7 +535,7 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }

source/client/src/clientSml.c

@@ -20,75 +20,93 @@
 
 #include "clientSml.h"
 
-int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
-int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
-int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
+#define RETURN_FALSE                                 \
+  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
+  return false;
 
-//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
-//  NodeList *tmp = list;
-//  while (tmp) {
-//    if (fn == NULL) {
-//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return tmp->data.value;
-//      }
-//    } else {
-//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
-//        return tmp->data.value;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  return NULL;
-//}
-//
-//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
-//  NodeList *tmp = *list;
-//  while (tmp) {
-//    if (!tmp->data.used) break;
-//    if (fn == NULL) {
-//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return -1;
-//      }
-//    } else {
-//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
-//        return -1;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  if (tmp) {
-//    tmp->data.key = key;
-//    tmp->data.keyLen = len;
-//    tmp->data.value = value;
-//    tmp->data.used = true;
-//  } else {
-//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
-//    if (newNode == NULL) {
-//      return -1;
-//    }
-//    newNode->data.key = key;
-//    newNode->data.keyLen = len;
-//    newNode->data.value = value;
-//    newNode->data.used = true;
-//    newNode->next = *list;
-//    *list = newNode;
-//  }
-//  return 0;
-//}
-//
-//int nodeListSize(NodeList *list) {
-//  int cnt = 0;
-//  while (list) {
-//    if (list->data.used)
-//      cnt++;
-//    else
-//      break;
-//    list = list->next;
-//  }
-//  return cnt;
-//}
+#define SET_DOUBLE                     \
+  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
+  kvVal->d = result;
+
+#define SET_FLOAT                                                                              \
+  if (!IS_VALID_FLOAT(result)) {                                                               \
+    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
+    return false;                                                                              \
+  }                                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
+  kvVal->f = (float)result;
+
+#define SET_BIGINT                                                                                       \
+  errno = 0;                                                                                             \
+  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
+  if (errno == ERANGE) {                                                                                 \
+    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
+    return false;                                                                                        \
+  }                                                                                                      \
+  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
+  kvVal->i = tmp;
+
+#define SET_INT                                                                    \
+  if (!IS_VALID_INT(result)) {                                                     \
+    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
+    return false;                                                                  \
+  }                                                                                \
+  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
+  kvVal->i = result;
+
+#define SET_SMALL_INT                                                          \
+  if (!IS_VALID_SMALLINT(result)) {                                            \
+    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
+    return false;                                                              \
+  }                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
+  kvVal->i = result;
+
+#define SET_UBIGINT                                                                             \
+  errno = 0;                                                                                    \
+  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
+  if (errno == ERANGE || result < 0) {                                                          \
+    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
+    return false;                                                                               \
+  }                                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
+  kvVal->u = tmp;
+
+#define SET_UINT                                                                  \
+  if (!IS_VALID_UINT(result)) {                                                   \
+    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
+  kvVal->u = result;
+
+#define SET_USMALL_INT                                                            \
+  if (!IS_VALID_USMALLINT(result)) {                                              \
+    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
+  kvVal->u = result;
+
+#define SET_TINYINT                                                       \
+  if (!IS_VALID_TINYINT(result)) {                                        \
+    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
+    return false;                                                         \
+  }                                                                       \
+  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
+  kvVal->i = result;
+
+#define SET_UTINYINT                                                            \
+  if (!IS_VALID_UTINYINT(result)) {                                             \
+    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
+    return false;                                                               \
+  }                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
+  kvVal->u = result;
+
+int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
+int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
 
 static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
   SUserAuthInfo pAuth = {0};
@@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
-inline bool smlDoubleToInt64OverFlow(double num) {
-  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
-  return false;
-}
-
-void smlStrReplace(char* src, int32_t len){
-  if (!tsSmlDot2Underline) return;
-  for(int i = 0; i < len; i++){
-    if(src[i] == '.'){
-      src[i] = '_';
-    }
-  }
-}
 
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
@@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
   return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
 }
 
-int8_t smlGetTsTypeByLen(int32_t len) {
-  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
-    return TSDB_TIME_PRECISION_SECONDS;
-  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-    return TSDB_TIME_PRECISION_MILLI;
-  } else {
-    return -1;
-  }
-}
-
 SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
   SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
   if (!tag) {
@@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu
     goto cleanup;
   }
 
-  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
-  //  if (tag->tags == NULL) {
-  //    uError("SML:smlBuildTableInfo failed to allocate memory");
-  //    goto cleanup;
-  //  }
   return tag;
 
 cleanup:
@@ -192,6 +182,242 @@ cleanup:
   return NULL;
 }
 
+void smlBuildTsKv(SSmlKv *kv, int64_t ts){
+  kv->key = tsSmlTsDefaultName;
+  kv->keyLen = strlen(tsSmlTsDefaultName);
+  kv->type = TSDB_DATA_TYPE_TIMESTAMP;
+  kv->i = ts;
+  kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){
+  SSmlSTableMeta* sMeta = NULL;
+  char *measure = currElement->measure;
+  int   measureLen = currElement->measureLen;
+  if (currElement->measureEscaped) {
+    measure = (char *)taosMemoryMalloc(measureLen);
+    memcpy(measure, currElement->measure, measureLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+  }
+  STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+  if (currElement->measureEscaped) {
+    taosMemoryFree(measure);
+  }
+  if (pTableMeta == NULL) {
+    info->dataFormat = false;
+    info->reRun = true;
+    terrno = TSDB_CODE_SUCCESS;
+    return sMeta;
+  }
+  sMeta = smlBuildSTableMeta(info->dataFormat);
+  if(sMeta == NULL){
+    taosMemoryFreeClear(pTableMeta);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return sMeta;
+  }
+  sMeta->tableMeta = pTableMeta;
+  taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+  for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
+    SSchema *col = pTableMeta->schema + i;
+    SSmlKv   kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
+    if (col->type == TSDB_DATA_TYPE_NCHAR) {
+      kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) {
+      kv.length = col->bytes - VARSTR_HEADER_SIZE;
+    } else{
+      kv.length = col->bytes;
+    }
+
+    if(i < pTableMeta->tableInfo.numOfColumns){
+      taosArrayPush(sMeta->cols, &kv);
+    }else{
+      taosArrayPush(sMeta->tags, &kv);
+    }
+  }
+  return sMeta;
+}
+
+bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  // cnt begin 0, add ts so + 2
+  if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  // bind data
+  int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
+  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+    uDebug("smlBuildCol error, retry");
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  if (cnt >= taosArrayGetSize(info->maxColKVs)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+
+  if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+}
+
+bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+    goto END;
+  }
+
+  if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    goto END;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    goto END;
+  }
+
+  if (unlikely(kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+
+END:
+  info->dataFormat = false;
+  info->reRun = true;
+  return false;
+}
+
+int32_t smlJoinMeasureTag(SSmlLineInfo *elements){
+  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+  if(elements->measureTag == NULL){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
+  bool isSameMeasure = IS_SAME_SUPER_TABLE;
+  if(isSameMeasure) {
+    return 0;
+  }
+  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+
+  SSmlSTableMeta *sMeta = NULL;
+  if (unlikely(tmp == NULL)) {
+    sMeta = smlBuildSuperTableInfo(info, elements);
+    if(sMeta == NULL) return -1;
+  }else{
+    sMeta = *tmp;
+  }
+  ASSERT(sMeta != NULL);
+  info->currSTableMeta = sMeta->tableMeta;
+  info->maxTagKVs = sMeta->tags;
+  info->maxColKVs = sMeta->cols;
+  return 0;
+}
+
+int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
+  SSmlTableInfo **oneTable =
+      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
+  SSmlTableInfo *tinfo = NULL;
+  if (unlikely(oneTable == NULL)) {
+    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
+    if (unlikely(!tinfo)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES);
+
+    tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
+    for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
+      SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
+      if (kv->keyEscaped) kv->key = NULL;
+      if (kv->valueEscaped) kv->value = NULL;
+    }
+
+    smlSetCTableName(tinfo);
+    getTableUid(info, elements, tinfo);
+    if (info->dataFormat) {
+      info->currSTableMeta->uid = tinfo->uid;
+      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
+      if (tinfo->tableDataCtx == NULL) {
+        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
+        smlDestroyTableInfo(&tinfo);
+        return TSDB_CODE_SML_INVALID_DATA;
+      }
+    }
+  }else{
+    tinfo = *oneTable;
+  }
+  ASSERT(tinfo != NULL);
+  if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1);
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i);
+    if (elements->colArray == NULL) {
+      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
+    }
+    taosArrayPush(elements->colArray, kvTs);
+    taosArrayPush(elements->colArray, kv);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i);
+    taosArraySet(elements->colArray, 0, kvTs);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
   bool autoChildName = false;
   size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
@@ -328,98 +554,6 @@ cleanup:
   return NULL;
 }
 
-// uint16_t smlCalTypeSum(char* endptr, int32_t left){
-//   uint16_t sum = 0;
-//   for(int i = 0; i < left; i++){
-//     sum += endptr[i];
-//   }
-//   return sum;
-// }
-
-#define RETURN_FALSE                                 \
-  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-  return false;
-
-#define SET_DOUBLE                     \
-  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-  kvVal->d = result;
-
-#define SET_FLOAT                                                                              \
-  if (!IS_VALID_FLOAT(result)) {                                                               \
-    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-    return false;                                                                              \
-  }                                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-  kvVal->f = (float)result;
-
-#define SET_BIGINT                                                                                       \
-  errno = 0;                                                                                             \
-  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-  if (errno == ERANGE) {                                                                                 \
-    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-    return false;                                                                                        \
-  }                                                                                                      \
-  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-  kvVal->i = tmp;
-
-#define SET_INT                                                                    \
-  if (!IS_VALID_INT(result)) {                                                     \
-    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-    return false;                                                                  \
-  }                                                                                \
-  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-  kvVal->i = result;
-
-#define SET_SMALL_INT                                                          \
-  if (!IS_VALID_SMALLINT(result)) {                                            \
-    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-    return false;                                                              \
-  }                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-  kvVal->i = result;
-
-#define SET_UBIGINT                                                                             \
-  errno = 0;                                                                                    \
-  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-  if (errno == ERANGE || result < 0) {                                                          \
-    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-    return false;                                                                               \
-  }                                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-  kvVal->u = tmp;
-
-#define SET_UINT                                                                  \
-  if (!IS_VALID_UINT(result)) {                                                   \
-    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-  kvVal->u = result;
-
-#define SET_USMALL_INT                                                            \
-  if (!IS_VALID_USMALLINT(result)) {                                              \
-    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-  kvVal->u = result;
-
-#define SET_TINYINT                                                       \
-  if (!IS_VALID_TINYINT(result)) {                                        \
-    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
-    return false;                                                         \
-  }                                                                       \
-  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
-  kvVal->i = result;
-
-#define SET_UTINYINT                                                            \
-  if (!IS_VALID_UTINYINT(result)) {                                             \
-    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
-    return false;                                                               \
-  }                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
-  kvVal->u = result;
-
 bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
   const char *pVal = kvVal->value;
   int32_t     len = kvVal->length;
@@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
 static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                               ESchemaAction action) {
   SRequestObj   *pRequest = NULL;
@@ -1121,35 +1253,6 @@ end:
   return code;
 }
 
-/*
-static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
-  for(int i = 0; i < taosArrayGetSize(tags); i++) {
-    SSmlKv *tag = taosArrayGet(tags, i);
-    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
-      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
-      return TSDB_CODE_TSC_DUP_NAMES;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
-  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-
-  end:
-  taosHashCleanup(dumplicateKey);
-  return ret;
-}
-*/
-
 static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) {
     taosHashCleanup(kvHash);
   }
 
-  //  if (info->parseJsonByLib) {
-  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
-  //    if (key != NULL) taosMemoryFree(key->tags);
-  //  }
-  //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
@@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
 
-  // destroy info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // destroy info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
-
-  // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
   taosHashCleanup(info->childTables);
   taosHashCleanup(info->superTables);
@@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) {
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
-      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+      if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) {
+        taosMemoryFree(info->lines[i].measureTag);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
         return ret;
       }
     } else {
-      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
-      //      if (ret != TSDB_CODE_SUCCESS) {
-      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
-      //        return ret;
-      //      }
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
@@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
   uDebug(
       "SML:0x%" PRIx64
       " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
-        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
-      "",
+        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64,
       info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
       info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
       info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
       info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
       info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
+
 }
 
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
-  // clear info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(info, *oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // clear info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);

source/client/src/clientSmlJson.c

@@ -29,199 +29,6 @@
       (start)++;                 \
   }
 
-// SArray *smlJsonParseTags(char *start, char *end){
-//   SArray *tags = taosArrayInit(4, sizeof(SSmlKv));
-//   while(start < end){
-//     SSmlKv kv = {0};
-//     kv.type = TSDB_DATA_TYPE_NCHAR;
-//     bool isInQuote = false;
-//     while(start < end){
-//       if(unlikely(!isInQuote && *start == '"')){
-//         start++;
-//         kv.key = start;
-//         isInQuote = true;
-//         continue;
-//       }
-//       if(unlikely(isInQuote && *start == '"')){
-//         kv.keyLen = start - kv.key;
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//     bool hasColon = false;
-//     while(start < end){
-//       if(unlikely(!hasColon && *start == ':')){
-//         start++;
-//         hasColon = true;
-//         continue;
-//       }
-//       if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){
-//         kv.value = start;
-//         start++;
-//         continue;
-//       }
-//
-//       if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){
-//         kv.length = start - kv.value;
-//         taosArrayPush(tags, &kv);
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//   }
-//   return tags;
-// }
-
-// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) {
-//   int32_t ret = TSDB_CODE_SUCCESS;
-//
-//   if(is_same_child_table_telnet(elements, &info->preLine) == 0){
-//     return TSDB_CODE_SUCCESS;
-//   }
-//
-//   bool isSameMeasure = IS_SAME_SUPER_TABLE;
-//
-//   int     cnt = 0;
-//   SArray *preLineKV = info->preLineTagKV;
-//   bool    isSuperKVInit = true;
-//   SArray *superKV = NULL;
-//   if(info->dataFormat){
-//     if(unlikely(!isSameMeasure)){
-//       SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//       elements->measureLen, NULL);
-//
-//       if(unlikely(sMeta == NULL)){
-//         sMeta = smlBuildSTableMeta(info->dataFormat);
-//         STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-//         sMeta->tableMeta = pTableMeta;
-//         if(pTableMeta == NULL){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
-//       }
-//       info->currSTableMeta = sMeta->tableMeta;
-//       superKV = sMeta->tags;
-//
-//       if(unlikely(taosArrayGetSize(superKV) == 0)){
-//         isSuperKVInit = false;
-//       }
-//       taosArraySetSize(preLineKV, 0);
-//     }
-//   }else{
-//     taosArraySetSize(preLineKV, 0);
-//   }
-//
-//   SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen);
-//   int32_t tagNum = taosArrayGetSize(tags);
-//   if (tagNum == 0) {
-//     uError("SML:tag is empty:%s", elements->tags)
-//     taosArrayDestroy(tags);
-//     return TSDB_CODE_SML_INVALID_DATA;
-//   }
-//   for (int32_t i = 0; i < tagNum; ++i) {
-//     SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i);
-//
-//     if(info->dataFormat){
-//       if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){
-//         info->dataFormat = false;
-//         info->reRun      = true;
-//         taosArrayDestroy(tags);
-//         return TSDB_CODE_SUCCESS;
-//       }
-//
-//       if(isSameMeasure){
-//         if(unlikely(cnt >= taosArrayGetSize(preLineKV))) {
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-//         if(unlikely(kv.length > preKV->length)){
-//           preKV->length = kv.length;
-//           SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//           elements->measureLen, NULL);
-//            if(unlikely(NULL == tableMeta)){
-//              uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-//              return TSDB_CODE_SML_INTERNAL_ERROR;
-//            }
-//
-//           SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
-//           oldKV->length = kv.length;
-//           info->needModifySchema = true;
-//         }
-//         if(unlikely(!IS_SAME_KEY)){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//       }else{
-//         if(isSuperKVInit){
-//           if(unlikely(cnt >= taosArrayGetSize(superKV))) {
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//           SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-//           if(unlikely(kv.length > preKV->length)) {
-//             preKV->length = kv.length;
-//           }else{
-//             kv.length = preKV->length;
-//           }
-//           info->needModifySchema = true;
-//
-//           if(unlikely(!IS_SAME_KEY)){
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//         }else{
-//           taosArrayPush(superKV, &kv);
-//         }
-//         taosArrayPush(preLineKV, &kv);
-//       }
-//     }else{
-//       taosArrayPush(preLineKV, &kv);
-//     }
-//     cnt++;
-//   }
-//   taosArrayDestroy(tags);
-//
-//   SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES,
-//   is_same_child_table_telnet); if (unlikely(tinfo == NULL)) {
-//     tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-//     if (unlikely(!tinfo)) {
-//       return TSDB_CODE_OUT_OF_MEMORY;
-//     }
-//     tinfo->tags = taosArrayDup(preLineKV, NULL);
-//
-//     smlSetCTableName(tinfo);
-//     if (info->dataFormat) {
-//       info->currSTableMeta->uid = tinfo->uid;
-//       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-//       if (tinfo->tableDataCtx == NULL) {
-//         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-//         return TSDB_CODE_SML_INVALID_DATA;
-//       }
-//     }
-//
-//     SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-//     *key = *elements;
-//     tinfo->key = key;
-//     nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
-//   }
-//   if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
-//
-//   return ret;
-// }
-
 static char *smlJsonGetObj(char *payload) {
   int  leftBracketCnt = 0;
   bool isInQuote = false;
@@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
       break;
     }
     case cJSON_String: {
-      /* set default JSON type to binary/nchar according to
-       * user configured parameter tsDefaultJSONStrType
-       */
-
-      char *tsDefaultJSONStrType = "binary";  // todo
-      smlConvertJSONString(kv, tsDefaultJSONStrType, root);
+      smlConvertJSONString(kv, "binary", root);
       break;
     }
     case cJSON_Object: {
@@ -682,138 +484,70 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
-  int32_t ret = TSDB_CODE_SUCCESS;
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   int32_t tagNum = cJSON_GetArraySize(tags);
   if (unlikely(tagNum == 0)) {
     uError("SML:Tag should not be empty");
-    return TSDB_CODE_TSC_INVALID_JSON;
+    terrno = TSDB_CODE_TSC_INVALID_JSON;
+    return -1;
   }
   for (int32_t i = 0; i < tagNum; ++i) {
     cJSON *tag = cJSON_GetArrayItem(tags, i);
     if (unlikely(tag == NULL)) {
-      return TSDB_CODE_TSC_INVALID_JSON;
+      terrno = TSDB_CODE_TSC_INVALID_JSON;
+      return -1;
     }
-    //    if(unlikely(tag == cMeasure)) continue;
     size_t keyLen = strlen(tag->string);
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
       uError("OTD:Tag key length is 0 or too large than 64");
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno =  TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // add kv to SSmlKv
-    SSmlKv kv = {.key = tag->string, .keyLen = keyLen};
+    SSmlKv kv = {0};
+    kv.key = tag->string;
+    kv.keyLen = keyLen;
+
     // value
-    ret = smlParseValueFromJSON(tag, &kv);
+    int32_t ret = smlParseValueFromJSON(tag, &kv);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      return ret;
-    }
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      terrno =  ret;
+      return -1;
     }
     taosArrayPush(preLineKV, &kv);
+
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno =  TSDB_CODE_SUCCESS;
+      return -1;
+    }
+
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (unlikely(!tinfo)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    if(info->parseJsonByLib){
-    //      key->tags = taosMemoryMalloc(elements->tagsLen + 1);
-    //      memcpy(key->tags, elements->tags, elements->tagsLen);
-    //      key->tags[elements->tagsLen] = 0;
-    //    }
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
-  return ret;
+  ret = smlProcessTagJson(info, tags);
+  if(ret != 0){
+    return terrno;
+  }
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+  return smlProcessChildTable(info, elements);
 }
 
 static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
@@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
@@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-
   if (unlikely(info->lines != NULL)) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
@@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 int32_t smlParseJSON(SSmlHandle *info, char *payload) {

source/client/src/clientSmlLine.c

@@ -20,15 +20,9 @@
 
 #include "clientSml.h"
 
-// comma ,
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-// space
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-// equal =
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
-// quote "
-// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-// SLASH
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
@@ -51,10 +45,10 @@
     }                                                \
   }
 
-#define BINARY_ADD_LEN 2  // "binary"   2 means " "
-#define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
+#define BINARY_ADD_LEN (sizeof("\"\"")-1)    // "binary"   2 means length of ("")
+#define NCHAR_ADD_LEN  (sizeof("L\"\"")-1)   // L"nchar"   3 means length of (L"")
 
-uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
+uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
                                   TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO,
                                   TSDB_TIME_PRECISION_NANO};
 
@@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   return TSDB_CODE_TSC_INVALID_VALUE;
 }
 
-static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
-  if (isSameCTable) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int     cnt = 0;
+static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
-        for (int i = pTableMeta->tableInfo.numOfColumns;
-             i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name,
-                         .keyLen = strlen(tag->name),
-                         .type = tag->type,
-                         .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE};
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
   taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
@@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     while (*sql < sqlEnd) {
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(IS_EQUAL(*sql))) {
         keyLen = *sql - key;
@@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // parse value
@@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         break;
       } else if (unlikely(IS_EQUAL(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(valueLen == 0)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_SML_INVALID_DATA;
+      terrno = TSDB_CODE_SML_INVALID_DATA;
+      return -1;
     }
 
     if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
     if (keyEscaped) {
@@ -331,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       value = tmp;
     }
     SSmlKv kv = {.key = key,
-                 .keyLen = keyLen,
-                 .type = TSDB_DATA_TYPE_NCHAR,
-                 .value = value,
-                 .length = valueLen,
-                 .keyEscaped = keyEscaped,
-                 .valueEscaped = valueEscaped};
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = keyEscaped,
+        .valueEscaped = valueEscaped};
     taosArrayPush(preLineKV, &kv);
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
 
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
     }
 
     cnt++;
@@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
     (*sql)++;
   }
+  return 0;
+}
 
-  void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-  if ((oneTable != NULL)) {
+static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
+  bool isSameCTable = IS_SAME_CHILD_TABLE;
+  if(isSameCTable){
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen);
-  if (unlikely(!tinfo)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tinfo->tags = taosArrayDup(preLineKV, NULL);
-  for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
-    if (kv->keyEscaped) kv->key = NULL;
-    if (kv->valueEscaped) kv->value = NULL;
-  }
-
-  smlSetCTableName(tinfo);
-  getTableUid(info, currElement, tinfo);
-  if (info->dataFormat) {
-    info->currSTableMeta->uid = tinfo->uid;
-    tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-    if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(&tinfo);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-      return TSDB_CODE_SML_INVALID_DATA;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
   }
 
-  taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
+  ret = smlProcessTagLine(info, sql, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
 
-  return TSDB_CODE_SUCCESS;
+  return smlProcessChildTable(info, elements);
 }
 
-static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
+static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
   int cnt = 0;
-  if (info->dataFormat) {
-    if (unlikely(!isSameCTable)) {
-      SSmlTableInfo **oneTable =
-          (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-      if (unlikely(oneTable == NULL)) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-      info->currTableDataCtx = (*oneTable)->tableDataCtx;
-    }
-
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        *tmp = smlBuildSTableMeta(info->dataFormat);
-        if(*tmp == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
-
-        for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
-          if (tag->type == TSDB_DATA_TYPE_NCHAR) {
-            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
-          } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
-            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
-          }
-          taosArrayPush((*tmp)->cols, &kv);
-        }
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxColKVs = (*tmp)->cols;
-    }
-  }
-
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
@@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
 
     if (info->dataFormat) {
-      // cnt begin 0, add ts so + 2
-      if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      // bind data
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1);
-      if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-        uDebug("smlBuildCol error, retry");
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (cnt >= taosArrayGetSize(info->maxColKVs)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
-      if (kv.type != maxKV->type) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      bool isAligned = isSmlColAligned(info, cnt, &kv);
       freeSSmlKv(&kv);
+      if(!isAligned){
+        return TSDB_CODE_SUCCESS;
+      }
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA;
   elements->measure = sql;
-
   // parse measure
   size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
@@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
-
-  bool isSameCTable = false;
-  bool isSameMeasure = false;
-  if (IS_SAME_CHILD_TABLE) {
-    isSameCTable = true;
-    isSameMeasure = true;
-  } else if (info->dataFormat) {
-    isSameMeasure = IS_SAME_SUPER_TABLE;
-  }
+  elements->measureTag = elements->measure;
   // parse tag
   if (*sql == COMMA) sql++;
   elements->tags = sql;
 
-  int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  int ret = smlParseTagLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   elements->cols = sql;
 
-  ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  ret = smlParseColLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  // add ts to
-  SSmlKv kv = {.key = tsSmlTsDefaultName,
-               .keyLen = strlen(tsSmlTsDefaultName),
-               .type = TSDB_DATA_TYPE_TIMESTAMP,
-               .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
-               .keyEscaped = false,
-               .valueEscaped = false};
-  if (info->dataFormat) {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
 
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
-    taosArraySet(elements->colArray, 0, &kv);
-  }
-  info->preLine = *elements;
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  return ret;
+  return smlParseEndLine(info, elements, &kvTs);
 }

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,6 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b) {
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  //  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-  //         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
     return 1;
   return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
@@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
   }
 }
 
-static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
-  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-    elements->measureTag = info->preLine.measureTag;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (!isSameMeasure) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
-  taosArrayClear(preLineKV);
   const char *sql = data;
   while (sql < sqlEnd) {
     JUMP_SPACE(sql, sqlEnd)
@@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     // parse key
     while (sql < sqlEnd) {
       if (unlikely(*sql == SPACE)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(*sql == EQUAL)) {
         keyLen = sql - key;
@@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     }
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
-      smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
-    //    if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) {
-    //      smlBuildInvalidDataMsg(msg, "dumplicate key", key);
-    //      return TSDB_CODE_TSC_DUP_NAMES;
-    //    }
 
     // parse value
     const char *value = sql;
@@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         break;
       }
       if (unlikely(*sql == EQUAL)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       sql++;
     }
     valueLen = sql - value;
 
     if (unlikely(valueLen == 0)) {
-      smlBuildInvalidDataMsg(msg, "invalid value", value);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
+      terrno = TSDB_CODE_TSC_INVALID_VALUE;
+      return -1;
     }
 
     if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-    }
+    SSmlKv kv = {.key = key,
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = false,
+        .valueEscaped = false};
     taosArrayPush(preLineKV, &kv);
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
+    }
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (!tinfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
+static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
+  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
+    elements->measureTag = info->preLine.measureTag;
+    return TSDB_CODE_SUCCESS;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
 
-  return TSDB_CODE_SUCCESS;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
+    }
+  }
+
+  ret = smlProcessTagTelnet(info, data, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+
+  return smlProcessChildTable(info, elements);
 }
 
 // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
@@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
-  bool needConverTime = false;  // get TS before parse tag(get meta), so need conver time
+  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
   if (info->dataFormat && info->currSTableMeta == NULL) {
     needConverTime = true;
   }
@@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
   // parse value
   smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
@@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf);
+  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SUCCESS;
   }
 
-  if (info->dataFormat && info->currSTableMeta != NULL) {
-    if (needConverTime) {
-      kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
-    }
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
+  if (needConverTime) {
+    kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
-  info->preLine = *elements;
 
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }

source/common/src/cos.c

@@ -1,6 +1,8 @@
 #define ALLOW_FORBID_FUNC
 
 #include "cos.h"
+#include "cos_cp.h"
+#include "tdef.h"
 
 extern char   tsS3Endpoint[];
 extern char   tsS3AccessKeyId[];
@@ -86,7 +88,7 @@ typedef struct {
   char     err_msg[128];
   S3Status status;
   uint64_t content_length;
-  char *   buf;
+  char    *buf;
   int64_t  buf_pos;
 } TS3SizeCBD;
 
@@ -270,7 +272,7 @@ typedef struct list_parts_callback_data {
 typedef struct MultipartPartData {
   put_object_callback_data put_object_data;
   int                      seq;
-  UploadManager *          manager;
+  UploadManager           *manager;
 } MultipartPartData;
 
 static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) {
@@ -317,12 +319,23 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti
 
   MultipartPartData *data = (MultipartPartData *)callbackData;
   int                seq = data->seq;
-  const char *       etag = properties->eTag;
+  const char        *etag = properties->eTag;
   data->manager->etags[seq - 1] = strdup(etag);
   data->manager->next_etags_pos = seq;
   return S3StatusOK;
 }
 
+S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) {
+  responsePropertiesCallbackNull(properties, callbackData);
+
+  MultipartPartData *data = (MultipartPartData *)callbackData;
+  int                seq = data->seq;
+  const char        *etag = properties->eTag;
+  data->manager->etags[seq - 1] = strdup(etag);
+  // data->manager->next_etags_pos = seq;
+  return S3StatusOK;
+}
+
 static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   int            ret = 0;
@@ -446,37 +459,343 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
   return 0;
 }
 */
-int32_t s3PutObjectFromFile2(const char *file, const char *object) {
-  int32_t     code = 0;
-  const char *key = object;
-  // const char              *uploadId = 0;
-  const char *             filename = 0;
+
+static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char const *object_name, int64_t size,
+                                         S3PutProperties *put_prop, put_object_callback_data *data) {
+  int32_t            code = 0;
+  S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  do {
+    S3_put_object(bucket_context, object_name, size, put_prop, 0, 0, &putObjectHandler, data);
+  } while (S3_status_is_retryable(data->status) && should_retry());
+
+  if (data->status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+  } else if (data->contentLength) {
+    uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
+    code = TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
+                                            int64_t contentLength, S3PutProperties *put_prop,
+                                            put_object_callback_data *data) {
+  int32_t       code = 0;
+  uint64_t      totalContentLength = contentLength;
+  uint64_t      todoContentLength = contentLength;
+  UploadManager manager = {0};
+
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
+  int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  const int max_part_num = 10000;
+  if (totalSeq > max_part_num) {
+    chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
+    totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  }
+
+  MultipartPartData partData;
+  memset(&partData, 0, sizeof(MultipartPartData));
+  int partContentLength = 0;
+
+  S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                       &initial_multipart_callback};
+
+  S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  S3MultipartCommitHandler commit_handler = {
+      {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
+  manager.next_etags_pos = 0;
+  do {
+    S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+
+  if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+upload:
+  todoContentLength -= chunk_size * manager.next_etags_pos;
+  for (int seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
+    partData.manager = &manager;
+    partData.seq = seq;
+    if (partData.put_object_data.gb == NULL) {
+      partData.put_object_data = *data;
+    }
+    partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
+    // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
+    partData.put_object_data.contentLength = partContentLength;
+    partData.put_object_data.originalContentLength = partContentLength;
+    partData.put_object_data.totalContentLength = todoContentLength;
+    partData.put_object_data.totalOriginalContentLength = totalContentLength;
+    put_prop->md5 = 0;
+    do {
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
+                     partContentLength, 0, timeoutMsG, &partData);
+    } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
+    if (partData.put_object_data.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+    contentLength -= chunk_size;
+    todoContentLength -= chunk_size;
+  }
+
+  int i;
+  int size = 0;
+  size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+  char buf[256];
+  int  n;
+  for (i = 0; i < totalSeq; i++) {
+    if (!manager.etags[i]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+    n = snprintf(buf, sizeof(buf),
+                 "<Part><PartNumber>%d</PartNumber>"
+                 "<ETag>%s</ETag></Part>",
+                 i + 1, manager.etags[i]);
+    size += growbuffer_append(&(manager.gb), buf, n);
+  }
+  size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+  manager.remaining = size;
+
+  do {
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                 timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+  if (manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+clean:
+  if (manager.upload_id) {
+    taosMemoryFree(manager.upload_id);
+  }
+  for (i = 0; i < manager.next_etags_pos; i++) {
+    taosMemoryFree(manager.etags[i]);
+  }
+  growbuffer_destroy(manager.gb);
+  taosMemoryFree(manager.etags);
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
+                                         char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
+                                         put_object_callback_data *data) {
+  int32_t code = 0;
+
+  uint64_t totalContentLength = contentLength;
+  // uint64_t      todoContentLength = contentLength;
+  UploadManager manager = {0};
+
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
+  int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  const int max_part_num = 10000;
+  if (totalSeq > max_part_num) {
+    chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
+    totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  }
+
+  bool need_init_upload = true;
+  char file_cp_path[TSDB_FILENAME_LEN];
+  snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
+
+  SCheckpoint cp = {0};
+  cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
+
+  if (taosCheckExistFile(file_cp_path)) {
+    if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
+      manager.upload_id = strdup(cp.upload_id);
+      need_init_upload = false;
+    } else {
+      cos_cp_remove(file_cp_path);
+    }
+  }
+
+  if (need_init_upload) {
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &initial_multipart_callback};
+    do {
+      S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
+    } while (S3_status_is_retryable(manager.status) && should_retry());
+
+    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
+  }
+
+  if (cos_cp_open(file_cp_path, &cp)) {
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  int     part_num = 0;
+  int64_t consume_bytes = 0;
+  // SCheckpointPart *parts = taosMemoryCalloc(cp.part_num, sizeof(SCheckpointPart));
+  // cos_cp_get_undo_parts(&cp, &part_num, parts, &consume_bytes);
+
+  MultipartPartData partData;
+  memset(&partData, 0, sizeof(MultipartPartData));
+  int partContentLength = 0;
+
+  S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallbackWithCp, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  S3MultipartCommitHandler commit_handler = {
+      {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
+  manager.next_etags_pos = 0;
+
+upload:
+  // todoContentLength -= chunk_size * manager.next_etags_pos;
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (cp.parts[i].completed) {
+      continue;
+    }
+
+    if (i > 0 && cp.parts[i - 1].completed) {
+      if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
+        code = TAOS_SYSTEM_ERROR(errno);
+        goto clean;
+      }
+    }
+
+    int seq = cp.parts[i].index + 1;
+
+    partData.manager = &manager;
+    partData.seq = seq;
+    if (partData.put_object_data.gb == NULL) {
+      partData.put_object_data = *data;
+    }
+
+    partContentLength = cp.parts[i].size;
+    partData.put_object_data.contentLength = partContentLength;
+    partData.put_object_data.originalContentLength = partContentLength;
+    // partData.put_object_data.totalContentLength = todoContentLength;
+    partData.put_object_data.totalOriginalContentLength = totalContentLength;
+    put_prop->md5 = 0;
+    do {
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
+                     partContentLength, 0, timeoutMsG, &partData);
+    } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
+    if (partData.put_object_data.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+
+      //(void)cos_cp_dump(&cp);
+      goto clean;
+    }
+
+    if (!manager.etags[seq - 1]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
+    (void)cos_cp_dump(&cp);
+
+    contentLength -= chunk_size;
+    // todoContentLength -= chunk_size;
+  }
+
+  cos_cp_close(cp.thefile);
+  cp.thefile = 0;
+
+  int size = 0;
+  size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+  char buf[256];
+  int  n;
+  for (int i = 0; i < cp.part_num; ++i) {
+    n = snprintf(buf, sizeof(buf),
+                 "<Part><PartNumber>%d</PartNumber>"
+                 "<ETag>%s</ETag></Part>",
+                 // i + 1, manager.etags[i]);
+                 cp.parts[i].index + 1, cp.parts[i].etag);
+    size += growbuffer_append(&(manager.gb), buf, n);
+  }
+  size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+  manager.remaining = size;
+
+  do {
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                 timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+  if (manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  cos_cp_remove(file_cp_path);
+
+clean:
+  /*
+  if (parts) {
+    taosMemoryFree(parts);
+  }
+  */
+  if (cp.thefile) {
+    cos_cp_close(cp.thefile);
+  }
+  if (cp.parts) {
+    taosMemoryFree(cp.parts);
+  }
+
+  if (manager.upload_id) {
+    taosMemoryFree(manager.upload_id);
+  }
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (manager.etags[i]) {
+      taosMemoryFree(manager.etags[i]);
+    }
+  }
+  taosMemoryFree(manager.etags);
+  growbuffer_destroy(manager.gb);
+
+  return code;
+}
+
+int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
+  int32_t                  code = 0;
+  int32_t                  lmtime = 0;
+  const char              *filename = 0;
   uint64_t                 contentLength = 0;
-  const char *             cacheControl = 0, *contentType = 0, *md5 = 0;
-  const char *             contentDispositionFilename = 0, *contentEncoding = 0;
+  const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
+  const char              *contentDispositionFilename = 0, *contentEncoding = 0;
   int64_t                  expires = -1;
   S3CannedAcl              cannedAcl = S3CannedAclPrivate;
   int                      metaPropertiesCount = 0;
   S3NameValue              metaProperties[S3_MAX_METADATA_COUNT];
   char                     useServerSideEncryption = 0;
   put_object_callback_data data = {0};
-  // int                      noStatus = 0;
 
-  // data.infile = 0;
-  // data.gb = 0;
-  // data.infileFD = NULL;
-  // data.noStatus = noStatus;
-
-  // uError("ERROR: %s stat file %s: ", __func__, file);
-  if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
-    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
+  if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
     return code;
   }
 
   if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
-    uError("ERROR: %s Failed to open file %s: ", __func__, file);
     code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open file %s: ", __func__, file);
     return code;
   }
 
@@ -493,143 +812,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
                                    metaProperties,  useServerSideEncryption};
 
   if (contentLength <= MULTIPART_CHUNK_SIZE) {
-    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
-                                           &putObjectDataCallback};
-
-    do {
-      S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
-    } while (S3_status_is_retryable(data.status) && should_retry());
-
-    if (data.status != S3StatusOK) {
-      s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-    } else if (data.contentLength) {
-      uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
-             (unsigned long long)data.contentLength);
-      code = TAOS_SYSTEM_ERROR(EIO);
-    }
+    code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
   } else {
-    uint64_t      totalContentLength = contentLength;
-    uint64_t      todoContentLength = contentLength;
-    UploadManager manager = {0};
-    // manager.upload_id = 0;
-    // manager.gb = 0;
-
-    // div round up
-    int       seq;
-    uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
-    int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
-    const int max_part_num = 10000;
-    if (totalSeq > max_part_num) {
-      chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
-      totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+    if (withcp) {
+      code = s3PutObjectFromFileWithCp(&bucketContext, file, lmtime, object_name, contentLength, &putProperties, &data);
+    } else {
+      code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
     }
-
-    MultipartPartData partData;
-    memset(&partData, 0, sizeof(MultipartPartData));
-    int partContentLength = 0;
-
-    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
-                                         &initial_multipart_callback};
-
-    S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
-                                           &putObjectDataCallback};
-
-    S3MultipartCommitHandler commit_handler = {
-        {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
-
-    manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
-    manager.next_etags_pos = 0;
-    /*
-    if (uploadId) {
-      manager.upload_id = strdup(uploadId);
-      manager.remaining = contentLength;
-      if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
-        fseek(data.infile, -(manager.remaining), 2);
-        taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
-        contentLength = manager.remaining;
-        goto upload;
-      } else {
-        goto clean;
-      }
-    }
-    */
-    do {
-      S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
-    } while (S3_status_is_retryable(manager.status) && should_retry());
-
-    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
-      s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-      goto clean;
-    }
-
-  upload:
-    todoContentLength -= chunk_size * manager.next_etags_pos;
-    for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
-      partData.manager = &manager;
-      partData.seq = seq;
-      if (partData.put_object_data.gb == NULL) {
-        partData.put_object_data = data;
-      }
-      partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
-      // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
-      partData.put_object_data.contentLength = partContentLength;
-      partData.put_object_data.originalContentLength = partContentLength;
-      partData.put_object_data.totalContentLength = todoContentLength;
-      partData.put_object_data.totalOriginalContentLength = totalContentLength;
-      putProperties.md5 = 0;
-      do {
-        S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
-                       partContentLength, 0, timeoutMsG, &partData);
-      } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
-      if (partData.put_object_data.status != S3StatusOK) {
-        s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
-        code = TAOS_SYSTEM_ERROR(EIO);
-        goto clean;
-      }
-      contentLength -= chunk_size;
-      todoContentLength -= chunk_size;
-    }
-
-    int i;
-    int size = 0;
-    size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
-    char buf[256];
-    int  n;
-    for (i = 0; i < totalSeq; i++) {
-      if (!manager.etags[i]) {
-        code = TAOS_SYSTEM_ERROR(EIO);
-        goto clean;
-      }
-      n = snprintf(buf, sizeof(buf),
-                   "<Part><PartNumber>%d</PartNumber>"
-                   "<ETag>%s</ETag></Part>",
-                   i + 1, manager.etags[i]);
-      size += growbuffer_append(&(manager.gb), buf, n);
-    }
-    size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
-    manager.remaining = size;
-
-    do {
-      S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
-                                   timeoutMsG, &manager);
-    } while (S3_status_is_retryable(manager.status) && should_retry());
-    if (manager.status != S3StatusOK) {
-      s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-      goto clean;
-    }
-
-  clean:
-    if (manager.upload_id) {
-      taosMemoryFree(manager.upload_id);
-    }
-    for (i = 0; i < manager.next_etags_pos; i++) {
-      taosMemoryFree(manager.etags[i]);
-    }
-    growbuffer_destroy(manager.gb);
-    taosMemoryFree(manager.etags);
   }
 
   if (data.infileFD) {
@@ -648,7 +837,7 @@ typedef struct list_bucket_callback_data {
   char     nextMarker[1024];
   int      keyCount;
   int      allDetails;
-  SArray * objectArray;
+  SArray  *objectArray;
 } list_bucket_callback_data;
 
 static S3Status listBucketCallback(int isTruncated, const char *nextMarker, int contentsCount,
@@ -693,11 +882,11 @@ static void s3FreeObjectKey(void *pItem) {
 
 static SArray *getListByPrefix(const char *prefix) {
   S3BucketContext     bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
-                                   0, awsRegionG};
+                                       0, awsRegionG};
   S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                            &listBucketCallback};
 
-  const char *              marker = 0, *delimiter = 0;
+  const char               *marker = 0, *delimiter = 0;
   int                       maxkeys = 0, allDetails = 0;
   list_bucket_callback_data data;
   data.objectArray = taosArrayInit(32, sizeof(void *));
@@ -738,7 +927,7 @@ static SArray *getListByPrefix(const char *prefix) {
 
 void s3DeleteObjects(const char *object_name[], int nobject) {
   S3BucketContext   bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
-                                   0, awsRegionG};
+                                     0, awsRegionG};
   S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
 
   for (int i = 0; i < nobject; ++i) {
@@ -789,7 +978,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
   const char *ifMatch = 0, *ifNotMatch = 0;
 
   S3BucketContext    bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
-                                   0, awsRegionG};
+                                      0, awsRegionG};
   S3GetConditions    getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
   S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback},
                                          &getObjectDataCallback};
@@ -827,7 +1016,7 @@ int32_t s3GetObjectToFile(const char *object_name, char *fileName) {
   const char *ifMatch = 0, *ifNotMatch = 0;
 
   S3BucketContext    bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
-                                   0, awsRegionG};
+                                      0, awsRegionG};
   S3GetConditions    getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch};
   S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                          &getObjectCallback};
@@ -858,7 +1047,7 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) {
   if (objectArray == NULL) return -1;
 
   for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) {
-    char *      object = taosArrayGetP(objectArray, i);
+    char       *object = taosArrayGetP(objectArray, i);
     const char *tmp = strchr(object, '/');
     tmp = (tmp == NULL) ? object : tmp + 1;
     char fileName[PATH_MAX] = {0};
@@ -949,12 +1138,12 @@ static void s3InitRequestOptions(cos_request_options_t *options, int is_cname) {
 
 int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
   int32_t                code = 0;
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   int                    is_cname = 0;
-  cos_status_t *         s = NULL;
+  cos_status_t          *s = NULL;
   cos_request_options_t *options = NULL;
   cos_string_t           bucket, object, file;
-  cos_table_t *          resp_headers;
+  cos_table_t           *resp_headers;
   // int                    traffic_limit = 0;
 
   cos_pool_create(&p, NULL);
@@ -983,18 +1172,19 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
   return code;
 }
 
-int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
+int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
   int32_t                     code = 0;
-  cos_pool_t *                p = NULL;
+  cos_pool_t                 *p = NULL;
   int                         is_cname = 0;
-  cos_status_t *              s = NULL;
-  cos_request_options_t *     options = NULL;
+  cos_status_t               *s = NULL;
+  cos_request_options_t      *options = NULL;
   cos_string_t                bucket, object, file;
-  cos_table_t *               resp_headers;
+  cos_table_t                *resp_headers;
   int                         traffic_limit = 0;
-  cos_table_t *               headers = NULL;
+  cos_table_t                *headers = NULL;
   cos_resumable_clt_params_t *clt_params = NULL;
 
+  (void)withcp;
   cos_pool_create(&p, NULL);
   options = cos_request_options_create(p);
   s3InitRequestOptions(options, is_cname);
@@ -1025,11 +1215,11 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
 }
 
 void s3DeleteObjectsByPrefix(const char *prefix_str) {
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   cos_request_options_t *options = NULL;
   int                    is_cname = 0;
   cos_string_t           bucket;
-  cos_status_t *         s = NULL;
+  cos_status_t          *s = NULL;
   cos_string_t           prefix;
 
   cos_pool_create(&p, NULL);
@@ -1044,10 +1234,10 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
 }
 
 void s3DeleteObjects(const char *object_name[], int nobject) {
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   int                    is_cname = 0;
   cos_string_t           bucket;
-  cos_table_t *          resp_headers = NULL;
+  cos_table_t           *resp_headers = NULL;
   cos_request_options_t *options = NULL;
   cos_list_t             object_list;
   cos_list_t             deleted_object_list;
@@ -1081,14 +1271,14 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
 
 bool s3Exists(const char *object_name) {
   bool                      ret = false;
-  cos_pool_t *              p = NULL;
+  cos_pool_t               *p = NULL;
   int                       is_cname = 0;
-  cos_status_t *            s = NULL;
-  cos_request_options_t *   options = NULL;
+  cos_status_t             *s = NULL;
+  cos_request_options_t    *options = NULL;
   cos_string_t              bucket;
   cos_string_t              object;
-  cos_table_t *             resp_headers;
-  cos_table_t *             headers = NULL;
+  cos_table_t              *resp_headers;
+  cos_table_t              *headers = NULL;
   cos_object_exist_status_e object_exist;
 
   cos_pool_create(&p, NULL);
@@ -1115,15 +1305,15 @@ bool s3Exists(const char *object_name) {
 
 bool s3Get(const char *object_name, const char *path) {
   bool                   ret = false;
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   int                    is_cname = 0;
-  cos_status_t *         s = NULL;
+  cos_status_t          *s = NULL;
   cos_request_options_t *options = NULL;
   cos_string_t           bucket;
   cos_string_t           object;
   cos_string_t           file;
-  cos_table_t *          resp_headers = NULL;
-  cos_table_t *          headers = NULL;
+  cos_table_t           *resp_headers = NULL;
+  cos_table_t           *headers = NULL;
   int                    traffic_limit = 0;
 
   //创建内存池
@@ -1159,15 +1349,15 @@ bool s3Get(const char *object_name, const char *path) {
 int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) {
   (void)check;
   int32_t                code = 0;
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   int                    is_cname = 0;
-  cos_status_t *         s = NULL;
+  cos_status_t          *s = NULL;
   cos_request_options_t *options = NULL;
   cos_string_t           bucket;
   cos_string_t           object;
-  cos_table_t *          resp_headers;
-  cos_table_t *          headers = NULL;
-  cos_buf_t *            content = NULL;
+  cos_table_t           *resp_headers;
+  cos_table_t           *headers = NULL;
+  cos_buf_t             *content = NULL;
   // cos_string_t file;
   // int  traffic_limit = 0;
   char range_buf[64];
@@ -1261,7 +1451,7 @@ void s3EvictCache(const char *path, long object_size) {
       terrno = TAOS_SYSTEM_ERROR(errno);
       vError("failed to open %s since %s", dir_name, terrstr());
     }
-    SArray *       evict_files = taosArrayInit(16, sizeof(SEvictFile));
+    SArray        *evict_files = taosArrayInit(16, sizeof(SEvictFile));
     tdbDirEntryPtr pDirEntry;
     while ((pDirEntry = taosReadDir(pDir)) != NULL) {
       char *name = taosGetDirEntryName(pDirEntry);
@@ -1303,13 +1493,13 @@ void s3EvictCache(const char *path, long object_size) {
 long s3Size(const char *object_name) {
   long size = 0;
 
-  cos_pool_t *           p = NULL;
+  cos_pool_t            *p = NULL;
   int                    is_cname = 0;
-  cos_status_t *         s = NULL;
+  cos_status_t          *s = NULL;
   cos_request_options_t *options = NULL;
   cos_string_t           bucket;
   cos_string_t           object;
-  cos_table_t *          resp_headers = NULL;
+  cos_table_t           *resp_headers = NULL;
 
   //创建内存池
   cos_pool_create(&p, NULL);
@@ -1344,7 +1534,7 @@ long s3Size(const char *object_name) {
 int32_t s3Init() { return 0; }
 void    s3CleanUp() {}
 int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
-int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
 void    s3DeleteObjectsByPrefix(const char *prefix) {}
 void    s3DeleteObjects(const char *object_name[], int nobject) {}
 bool    s3Exists(const char *object_name) { return false; }

source/common/src/cos_cp.c

@@ -0,0 +1,417 @@
+#define ALLOW_FORBID_FUNC
+
+#include "cos_cp.h"
+#include "cJSON.h"
+#include "tutil.h"
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, cp_path);
+    return code;
+  }
+
+  checkpoint->thefile = fd;
+
+  return code;
+}
+
+void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
+void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
+
+static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
+  int32_t      code = 0;
+  cJSON const* item2 = NULL;
+
+  cJSON* json = cJSON_Parse(cp_body);
+  if (NULL == json) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+
+  cJSON const* item = cJSON_GetObjectItem(json, "ver");
+  if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
+    goto _exit;
+  }
+
+  item = cJSON_GetObjectItem(json, "type");
+  if (!cJSON_IsNumber(item)) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+  cp->cp_type = item->valuedouble;
+
+  item = cJSON_GetObjectItem(json, "md5");
+  if (cJSON_IsString(item)) {
+    memcpy(cp->md5, item->valuestring, strlen(item->valuestring));
+  }
+
+  item = cJSON_GetObjectItem(json, "upload_id");
+  if (cJSON_IsString(item)) {
+    strncpy(cp->upload_id, item->valuestring, 128);
+  }
+
+  item2 = cJSON_GetObjectItem(json, "file");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->file_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "lastmodified");
+    if (cJSON_IsNumber(item)) {
+      cp->file_last_modified = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "path");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_path, item->valuestring, TSDB_FILENAME_LEN);
+    }
+
+    item = cJSON_GetObjectItem(item2, "file_md5");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_md5, item->valuestring, 64);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "object");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "object_size");
+    if (cJSON_IsNumber(item)) {
+      cp->object_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_name");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_name, item->valuestring, 128);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_last_modified");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_last_modified, item->valuestring, 64);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_etag");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_etag, item->valuestring, 128);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "cpparts");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "number");
+    if (cJSON_IsNumber(item)) {
+      cp->part_num = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->part_size = item->valuedouble;
+    }
+
+    item2 = cJSON_GetObjectItem(item2, "parts");
+    if (cJSON_IsArray(item2) && cp->part_num > 0) {
+      cJSON_ArrayForEach(item, item2) {
+        cJSON const* item3 = cJSON_GetObjectItem(item, "index");
+        int32_t      index = 0;
+        if (cJSON_IsNumber(item3)) {
+          index = item3->valuedouble;
+          cp->parts[index].index = index;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "offset");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].offset = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "size");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].size = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "completed");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].completed = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "crc64");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].crc64 = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "etag");
+        if (cJSON_IsString(item3)) {
+          strncpy(cp->parts[index].etag, item3->valuestring, 128);
+        }
+      }
+    }
+  }
+
+_exit:
+  if (json) cJSON_Delete(json);
+  if (cp_body) taosMemoryFree(cp_body);
+
+  return code;
+}
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, filepath);
+    goto _exit;
+  }
+
+  int64_t size = -1;
+  code = taosStatFile(filepath, &size, NULL, NULL);
+  if (code) {
+    uError("ERROR: %s Failed to stat %s", __func__, filepath);
+    goto _exit;
+  }
+
+  char* cp_body = taosMemoryMalloc(size + 1);
+
+  int64_t n = taosReadFile(fd, cp_body, size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to read %s", __func__, filepath);
+    goto _exit;
+  } else if (n != size) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
+    goto _exit;
+  }
+  taosCloseFile(&fd);
+  cp_body[size] = '\0';
+
+  return cos_cp_parse_body(cp_body, checkpoint);
+
+_exit:
+  if (fd) {
+    taosCloseFile(&fd);
+  }
+  if (cp_body) {
+    taosMemoryFree(cp_body);
+  }
+
+  return code;
+}
+
+static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  char* data = cJSON_PrintUnformatted(json);
+  if (NULL == data) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  TdFilePtr fp = checkpoint->thefile;
+  if (taosFtruncateFile(fp, 0) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosWriteFile(fp, data, strlen(data)) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+  if (taosFsyncFile(fp) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+_exit:
+  taosMemoryFree(data);
+  return code;
+}
+
+int32_t cos_cp_dump(SCheckpoint* cp) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  cJSON* ojson = NULL;
+  cJSON* json = cJSON_CreateObject();
+  if (!json) return TSDB_CODE_OUT_OF_MEMORY;
+
+  if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "file");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "object");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  ojson = cJSON_AddObjectToObject(json, "cpparts");
+  if (!ojson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
+  if (!ajson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  for (int i = 0; i < cp->part_num; ++i) {
+    cJSON* item = cJSON_CreateObject();
+    if (!item) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    cJSON_AddItemToArray(ajson, item);
+
+    if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  code = cos_cp_save_json(json, cp);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+  }
+  cJSON_Delete(json);
+  return code;
+}
+
+void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}
+
+void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
+  checkpoint->parts[part_index].completed = 1;
+  strncpy(checkpoint->parts[part_index].etag, etag, 128);
+  checkpoint->parts[part_index].crc64 = crc64;
+}
+
+void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                         char const* upload_id, int64_t part_size) {
+  int i = 0;
+
+  checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
+  strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
+
+  checkpoint->file_size = size;
+  checkpoint->file_last_modified = mtime;
+  strncpy(checkpoint->upload_id, upload_id, 128);
+
+  checkpoint->part_size = part_size;
+  for (; i * part_size < size; i++) {
+    checkpoint->parts[i].index = i;
+    checkpoint->parts[i].offset = i * part_size;
+    checkpoint->parts[i].size = TMIN(part_size, (size - i * part_size));
+    checkpoint->parts[i].completed = 0;
+    checkpoint->parts[i].etag[0] = '\0';
+  }
+  checkpoint->part_num = i;
+}
+
+static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
+
+bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
+  if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
+    return true;
+  }
+
+  return false;
+}

source/common/src/tdataformat.c

@@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
     n += tPutCStr(p ? p + n : p, pTagVal->pKey);
   } else {
     n += tPutI16v(p ? p + n : p, pTagVal->cid);
+    ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
   }
 
   // type

source/common/src/tglobal.c

@@ -95,8 +95,9 @@ int32_t  tsMonitorMaxLogs = 100;
 bool     tsMonitorComp = false;
 
 // audit
-bool tsEnableAudit = true;
-bool tsEnableAuditCreateTable = true;
+bool     tsEnableAudit = true;
+bool     tsEnableAuditCreateTable = true;
+int32_t  tsAuditInterval = 5000;
 
 // telem
 #ifdef TD_ENTERPRISE
@@ -686,6 +687,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  if (cfgAddInt32(pCfg, "auditInterval", tsAuditInterval, 500, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
+    return -1;
 
   if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1;
@@ -1137,6 +1140,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsEnableAudit = cfgGetItem(pCfg, "audit")->bval;
   tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval;
+  tsAuditInterval = cfgGetItem(pCfg, "auditInterval")->i32;
 
   tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval;
   tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval;

source/common/src/ttime.c

@@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
     i += 2;
   }
 
+  if (hour > 12 || hour < 0) {
+    return -1;
+  }
+
   // return error if there're illegal charaters after min(2 Digits)
   char* minStr = &str[i];
   if (minStr[1] != '\0' && minStr[2] != '\0') {
@@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
   }
 
   int64_t minute = strnatoi(&str[i], 2);
-  if (minute > 59) {
+  if (minute > 59 || (hour == 12 && minute > 0)) {
     return -1;
   }
 

source/common/src/tvariant.c

@@ -19,6 +19,278 @@
 #include "ttokendef.h"
 #include "tvariant.h"
 
+int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
+  // skip head 0b
+  const char *p = z + 2;
+  int32_t     l = n - 2;
+
+  while (*p == '0' && l > 0) {
+    p++;
+    l--;
+  }
+  if (l > 64) {  // too big
+    return TSDB_CODE_FAILED;
+  }
+
+  uint64_t val = 0;
+  for (int32_t i = 0; i < l; i++) {
+    val = val << 1;
+    if (p[i] == '1') {
+      val |= 1;
+    } else if (p[i] != '0') {  // abnormal char
+      return TSDB_CODE_FAILED;
+    }
+  }
+  *value = val;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) {
+  // parse sign
+  bool has_sign = false;
+  if (z[0] == '-') {
+    *is_neg = true;
+    has_sign = true;
+  } else if (z[0] == '+') {
+    has_sign = true;
+  } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) {
+    return TSDB_CODE_FAILED;
+  }
+  if (has_sign) {
+    if (n < 2) {
+      return TSDB_CODE_FAILED;
+    }
+    z++;
+    n--;
+  }
+
+  errno = 0;
+  char  *endPtr = NULL;
+  bool parsed = false;
+  if (z[0] == '0' && n > 2) {
+    if (z[1] == 'b' || z[1] == 'B') {
+      // paring as binary
+      return parseBinaryUInteger(z, n, value);
+    }
+
+    if (z[1] == 'x' || z[1] == 'X') {
+      // parsing as hex
+      *value = taosStr2UInt64(z, &endPtr, 16);
+      parsed = true;
+    }
+  }
+
+  if (!parsed) {
+    // parsing as double
+    double val = taosStr2Double(z, &endPtr);
+    if (val > UINT64_MAX) {
+      return TSDB_CODE_FAILED;
+    }
+    *value = round(val);
+  }
+
+  if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) {
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  errno = 0;
+  char* endPtr = NULL;
+  *value = taosStr2Double(z, &endPtr);
+
+  if (errno == ERANGE || errno == EINVAL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  switch (type)
+  {
+    case TK_NK_INTEGER: {
+      *value = taosStr2Int64(z, &endPtr, 10);
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(z, &endPtr));
+      if (!IS_VALID_INT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+  default:
+    break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. try to parse as integer
+  *value = taosStr2Int64(z, &endPtr, 10);
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    // pure decimal part
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr+1 < end && endPtr[1] > '4') {
+        const char *p = z;
+        while (*p == ' ') {
+          p++;
+        }
+        if (*p == '-') {
+          if ( *value > INT64_MIN) {
+            (*value)--;
+          }
+        } else {
+          if ( *value < INT64_MAX) {
+            (*value)++;
+          }
+        }
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool     is_neg = false;
+  uint64_t uv = 0;
+  int32_t  code = parseSignAndUInteger(z, n, &is_neg, &uv);
+  if (code == TSDB_CODE_SUCCESS) {
+    // truncate into int64
+    if (is_neg) {
+      if (uv > 1ull + INT64_MAX) {
+        *value = INT64_MIN;
+        return TSDB_CODE_FAILED;
+      } else {
+        *value = -uv;
+      }
+    } else {
+      if (uv > INT64_MAX) {
+        *value = INT64_MAX;
+        return TSDB_CODE_FAILED;
+      }
+      *value = uv;
+    }
+  }
+
+  return code;
+}
+
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  const char *p = z;
+  while (*p == ' ') {
+    p++;
+  }
+  switch (type) {
+    case TK_NK_INTEGER: {
+      *value = taosStr2UInt64(p, &endPtr, 10);
+      if (*p == '-' && *value) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(p, &endPtr));
+      if (!IS_VALID_UINT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+    default:
+      break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. parse as integer
+  *value = taosStr2UInt64(p, &endPtr, 10);
+  if (*p == '-' && *value) {
+    return TSDB_CODE_FAILED;
+  }
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) {
+        (*value)++;
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool    is_neg = false;
+  int32_t code = parseSignAndUInteger(z, n, &is_neg, value);
+  if (is_neg) {
+    if (TSDB_CODE_SUCCESS == code && 0 == *value) {
+      return TSDB_CODE_SUCCESS;
+    }
+    return TSDB_CODE_FAILED;
+  }
+  return code;
+}
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   errno = 0;
   char *endPtr = NULL;
@@ -26,10 +298,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   *value = taosStr2Int64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
@@ -39,16 +311,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
   const char *p = z;
   while (*p == ' ') p++;
   if (*p == '-') {
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
   *value = taosStr2UInt64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
-
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -147,14 +418,13 @@ void taosVariantDestroy(SVariant *pVar) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
-
 }
 
 void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (pSrc == NULL || pDst == NULL) return;
 
   pDst->nType = pSrc->nType;
-  if (pSrc->nType == TSDB_DATA_TYPE_BINARY ||pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
+  if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_VARBINARY ||
       pSrc->nType == TSDB_DATA_TYPE_NCHAR || pSrc->nType == TSDB_DATA_TYPE_JSON ||
       pSrc->nType == TSDB_DATA_TYPE_GEOMETRY) {
     int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
@@ -172,7 +442,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i = pSrc->i;
   }
-
 }
 
 int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {

source/common/test/commonTests.cpp

@@ -14,6 +14,7 @@
 #include "tdef.h"
 #include "tvariant.h"
 #include "ttime.h"
+#include "ttokendef.h"
 
 namespace {
 //
@@ -24,40 +25,177 @@ int main(int argc, char** argv) {
   return RUN_ALL_TESTS();
 }
 
-TEST(testCase, toInteger_test) {
+
+TEST(testCase, toUIntegerEx_test) {
+  uint64_t val = 0;
+
   char*    s = "123";
-  uint32_t type = 0;
-
-  int64_t val = 0;
-
-  int32_t ret = toInteger(s, strlen(s), 10, &val);
+  int32_t ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 123);
 
+  s = "1000u";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "0b110";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "2567.4787";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2567);
+
+  s = "1.869895343e4";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  s = "-1";
+  ret = toUIntegerEx(s, strlen(s),TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0b10010";
+  ret = toUIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-0x40";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-80.9999";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-5.2343544534e10";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+
+  // INT64_MAX
   s = "9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 9223372036854775807);
 
-  s = "9323372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  // UINT64_MAX
+  s = "18446744073709551615";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 9323372036854775807u);
+  ASSERT_EQ(val, 18446744073709551615u);
+
+  // out of range
+  s = "18446744073709551616";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "5.23e25";
+  ret = toUIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toIntegerEx_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
+  s = "-1";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -1);
+
+  s = "1000u";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "0x1f";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 31);
+
+  s = "-0x40";
+  ret = toIntegerEx(s, strlen(s), TK_NK_HEX, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -64);
+
+  s = "0b110";
+  ret = toIntegerEx(s, strlen(s), TK_NK_BIN, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 6);
+
+  s = "-0b10010";
+  ret = toIntegerEx(s, strlen(s), 0, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -18);
+
+  s = "-80.9999";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -81);
+
+  s = "2567.8787";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 2568);
+
+  s = "-5.2343544534e10";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -52343544534);
+
+  s = "1.869895343e4";
+  ret = toIntegerEx(s, strlen(s), TK_NK_FLOAT, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 18699);
+
+  // INT64_MAX
+  s = "9223372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 9223372036854775807LL);
+
+  s = "-9223372036854775808";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
 
   s = "-9323372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
   ASSERT_EQ(ret, -1);
 
+  // UINT64_MAX
+  s = "18446744073709551615";
+  ret = toIntegerEx(s, strlen(s), TK_NK_INTEGER, &val);
+  ASSERT_EQ(ret, -1);
+}
+
+TEST(testCase, toInteger_test) {
+  int64_t val = 0;
+
+  char*    s = "123";
+  int32_t ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, 123);
+
   s = "-1";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, -1);
 
-  s = "-9223372036854775807";
-  ret = toInteger(s, strlen(s), 10, &val);
-  ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, -9223372036854775807);
-
   s = "1000u";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
@@ -77,13 +215,22 @@ TEST(testCase, toInteger_test) {
   ASSERT_EQ(ret, 0);
   ASSERT_EQ(val, 72);
 
-  // 18446744073709551615  UINT64_MAX
-  s = "18446744073709551615";
+  s = "9223372036854775807";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, 0);
-  ASSERT_EQ(val, 18446744073709551615u);
+  ASSERT_EQ(val, 9223372036854775807);
 
-  s = "18446744073709551616";
+  s = "-9223372036854775808";
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, 0);
+  ASSERT_EQ(val, -9223372036854775808);
+
+  // out of range
+  s = "9323372036854775807"; 
+  ret = toInteger(s, strlen(s), 10, &val);
+  ASSERT_EQ(ret, -1);
+
+  s = "-9323372036854775807";
   ret = toInteger(s, strlen(s), 10, &val);
   ASSERT_EQ(ret, -1);
 }

source/dnode/mgmt/mgmt_dnode/inc/dmInt.h

@@ -30,12 +30,14 @@ typedef struct SDnodeMgmt {
   TdThread               statusThread;
   TdThread               notifyThread;
   TdThread               monitorThread;
+  TdThread               auditThread;
   TdThread               crashReportThread;
   SSingleWorker          mgmtWorker;
   ProcessCreateNodeFp    processCreateNodeFp;
   ProcessAlterNodeTypeFp processAlterNodeTypeFp;
   ProcessDropNodeFp      processDropNodeFp;
   SendMonitorReportFp    sendMonitorReportFp;
+  SendAuditRecordsFp     sendAuditRecordsFp;
   GetVnodeLoadsFp        getVnodeLoadsFp;
   GetVnodeLoadsFp        getVnodeLoadsLiteFp;
   GetMnodeLoadsFp        getMnodeLoadsFp;
@@ -62,7 +64,9 @@ void    dmStopStatusThread(SDnodeMgmt *pMgmt);
 int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt);
 void    dmStopNotifyThread(SDnodeMgmt *pMgmt);
 int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt);
+int32_t dmStartAuditThread(SDnodeMgmt *pMgmt);
 void    dmStopMonitorThread(SDnodeMgmt *pMgmt);
+void    dmStopAuditThread(SDnodeMgmt *pMgmt);
 int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt);
 void    dmStopCrashReportThread(SDnodeMgmt *pMgmt);
 int32_t dmStartWorker(SDnodeMgmt *pMgmt);

source/dnode/mgmt/mgmt_dnode/src/dmInt.c

@@ -29,6 +29,9 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
   if (dmStartMonitorThread(pMgmt) != 0) {
     return -1;
   }
+  if (dmStartAuditThread(pMgmt) != 0) {
+    return -1;
+  }
   if (dmStartCrashReportThread(pMgmt) != 0) {
     return -1;
   }
@@ -38,6 +41,7 @@ static int32_t dmStartMgmt(SDnodeMgmt *pMgmt) {
 static void dmStopMgmt(SDnodeMgmt *pMgmt) {
   pMgmt->pData->stopped = true;
   dmStopMonitorThread(pMgmt);
+  dmStopAuditThread(pMgmt);
   dmStopStatusThread(pMgmt);
 #if defined(TD_ENTERPRISE)
   dmStopNotifyThread(pMgmt);
@@ -60,6 +64,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
   pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp;
   pMgmt->processDropNodeFp = pInput->processDropNodeFp;
   pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp;
+  pMgmt->sendAuditRecordsFp = pInput->sendAuditRecordFp;
   pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp;
   pMgmt->getVnodeLoadsLiteFp = pInput->getVnodeLoadsLiteFp;
   pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;

source/dnode/mgmt/mgmt_dnode/src/dmWorker.c

@@ -99,6 +99,27 @@ static void *dmMonitorThreadFp(void *param) {
   return NULL;
 }
 
+static void *dmAuditThreadFp(void *param) {
+  SDnodeMgmt *pMgmt = param;
+  int64_t     lastTime = taosGetTimestampMs();
+  setThreadName("dnode-audit");
+
+  while (1) {
+    taosMsleep(100);
+    if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
+
+    int64_t curTime = taosGetTimestampMs();
+    if (curTime < lastTime) lastTime = curTime;
+    float interval = curTime - lastTime;
+    if (interval >= tsAuditInterval) {
+      (*pMgmt->sendAuditRecordsFp)();
+      lastTime = curTime;
+    }
+  }
+
+  return NULL;
+}
+
 static void *dmCrashReportThreadFp(void *param) {
   SDnodeMgmt *pMgmt = param;
   int64_t     lastTime = taosGetTimestampMs();
@@ -218,6 +239,20 @@ int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {
   return 0;
 }
 
+int32_t dmStartAuditThread(SDnodeMgmt *pMgmt) {
+  TdThreadAttr thAttr;
+  taosThreadAttrInit(&thAttr);
+  taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
+  if (taosThreadCreate(&pMgmt->auditThread, &thAttr, dmAuditThreadFp, pMgmt) != 0) {
+    dError("failed to create audit thread since %s", strerror(errno));
+    return -1;
+  }
+
+  taosThreadAttrDestroy(&thAttr);
+  tmsgReportStartup("dnode-audit", "initialized");
+  return 0;
+}
+
 void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
   if (taosCheckPthreadValid(pMgmt->monitorThread)) {
     taosThreadJoin(pMgmt->monitorThread, NULL);
@@ -225,6 +260,13 @@ void dmStopMonitorThread(SDnodeMgmt *pMgmt) {
   }
 }
 
+void dmStopAuditThread(SDnodeMgmt *pMgmt) {
+  if (taosCheckPthreadValid(pMgmt->auditThread)) {
+    taosThreadJoin(pMgmt->auditThread, NULL);
+    taosThreadClear(&pMgmt->auditThread);
+  }
+}
+
 int32_t dmStartCrashReportThread(SDnodeMgmt *pMgmt) {
   if (!tsEnableCrashReport) {
     return 0;

source/dnode/mgmt/node_mgmt/inc/dmMgmt.h

@@ -70,6 +70,21 @@ typedef struct SUdfdData {
   int32_t      dnodeId;
 } SUdfdData;
 
+#ifndef TD_MODULE_OPTIMIZE
+typedef struct SDnode {
+  int8_t        once;
+  bool          stop;
+  EDndRunStatus status;
+  SStartupInfo  startup;
+  SDnodeData    data;
+  SUdfdData     udfdData;
+  TdThreadMutex mutex;
+  TdFilePtr     lockfile;
+  STfs         *pTfs;
+  SMgmtWrapper  wrappers[NODE_END];
+  SDnodeTrans   trans;
+} SDnode;
+#else
 typedef struct SDnode {
   int8_t        once;
   bool          stop;
@@ -83,6 +98,7 @@ typedef struct SDnode {
   STfs         *pTfs;
   SMgmtWrapper  wrappers[NODE_END];
 } SDnode;
+#endif
 
 // dmEnv.c
 SDnode *dmInstance();
@@ -124,6 +140,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 
 // dmMonitor.c
 void dmSendMonitorReport();
+void dmSendAuditRecords();
 void dmGetVnodeLoads(SMonVloadInfo *pInfo);
 void dmGetVnodeLoadsLite(SMonVloadInfo *pInfo);
 void dmGetMnodeLoads(SMonMloadInfo *pInfo);

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -189,6 +189,7 @@ void dmCleanup() {
   if (dmCheckRepeatCleanup(pDnode) != 0) return;
   dmCleanupDnode(pDnode);
   monCleanup();
+  auditCleanup();
   syncCleanUp();
   walCleanUp();
   udfcClose();
@@ -396,6 +397,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) {
       .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq,
       .processDropNodeFp = dmProcessDropNodeReq,
       .sendMonitorReportFp = dmSendMonitorReport,
+      .sendAuditRecordFp = auditSendRecordsInBatch,
       .getVnodeLoadsFp = dmGetVnodeLoads,
       .getVnodeLoadsLiteFp = dmGetVnodeLoadsLite,
       .getMnodeLoadsFp = dmGetMnodeLoads,

source/dnode/mgmt/node_mgmt/src/dmMonitor.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "dmMgmt.h"
 #include "dmNodes.h"
+#include "audit.h"
 
 static void dmGetMonitorBasicInfo(SDnode *pDnode, SMonBasicInfo *pInfo) {
   pInfo->protocol = 1;
@@ -108,6 +109,11 @@ void dmSendMonitorReport() {
   monSendReport();
 }
 
+//Todo: put this in seperate file in the future
+void dmSendAuditRecords() {
+  auditSendRecordsInBatch();
+}
+
 void dmGetVnodeLoads(SMonVloadInfo *pInfo) {
   SDnode       *pDnode = dmInstance();
   SMgmtWrapper *pWrapper = &pDnode->wrappers[VNODE];

source/dnode/mgmt/node_util/inc/dmUtil.h

@@ -86,6 +86,7 @@ typedef enum {
 typedef int32_t (*ProcessCreateNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
 typedef int32_t (*ProcessDropNodeFp)(EDndNodeType ntype, SRpcMsg *pMsg);
 typedef void (*SendMonitorReportFp)();
+typedef void (*SendAuditRecordsFp)();
 typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo);
 typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo);
 typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo);
@@ -120,6 +121,7 @@ typedef struct {
   ProcessAlterNodeTypeFp processAlterNodeTypeFp;
   ProcessDropNodeFp   processDropNodeFp;
   SendMonitorReportFp sendMonitorReportFp;
+  SendAuditRecordsFp  sendAuditRecordFp;
   GetVnodeLoadsFp     getVnodeLoadsFp;
   GetVnodeLoadsFp     getVnodeLoadsLiteFp;
   GetMnodeLoadsFp     getMnodeLoadsFp;

source/dnode/mgmt/test/qnode/dqnode.cpp

@@ -37,6 +37,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -50,6 +51,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -63,6 +65,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   test.Restart();
@@ -78,6 +81,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -94,6 +98,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 #endif
 
@@ -108,6 +113,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -121,6 +127,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   test.Restart();
@@ -136,6 +143,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -149,5 +157,6 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }

source/dnode/mgmt/test/snode/dsnode.cpp

@@ -37,6 +37,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -50,6 +51,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -63,6 +65,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   test.Restart();
@@ -78,6 +81,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -94,6 +98,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 #endif
 
@@ -108,6 +113,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -121,6 +127,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   test.Restart();
@@ -136,6 +143,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -149,5 +157,6 @@ TEST_F(DndTestSnode, 01_Drop_Snode) {
     SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }

source/dnode/mnode/impl/src/mndStream.c

@@ -704,10 +704,10 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask
   pReq->streamId = pTask->id.streamId;
 
   STransAction action = {0};
-  SEpSet  epset = {0};
-  if(pTask->info.nodeId == SNODE_HANDLE){
+  SEpSet       epset = {0};
+  if (pTask->info.nodeId == SNODE_HANDLE) {
     SSnodeObj *pObj = NULL;
-    void  *pIter = NULL;
+    void      *pIter = NULL;
     while (1) {
       pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj);
       if (pIter == NULL) {
@@ -717,10 +717,16 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask
       addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port);
       sdbRelease(pMnode->pSdb, pObj);
     }
-  }else{
+  } else {
     SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId);
-    epset = mndGetVgroupEpset(pMnode, pVgObj);
-    mndReleaseVgroup(pMnode, pVgObj);
+    if (pVgObj != NULL) {
+      epset = mndGetVgroupEpset(pMnode, pVgObj);
+      mndReleaseVgroup(pMnode, pVgObj);
+    } else {
+      mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", pTask->id.taskId, pTask->info.nodeId);
+      taosMemoryFree(pReq);
+      return 0;
+    }
   }
 
   // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
@@ -1657,6 +1663,7 @@ static void setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SSDat
 
   STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
   if (pe == NULL) {
+    mError("task:0x%" PRIx64 " not exists in vnode, no valid status/stage info", id.taskId);
     return;
   }
 

source/dnode/mnode/impl/src/mndStreamTrans.c

@@ -48,8 +48,7 @@ int32_t clearFinishedTrans(SMnode* pMnode) {
       void* pKey = taosHashGetKey(pEntry, &keyLen);
       // key is the name of src/dst db name
       SKeyInfo info = {.pKey = pKey, .keyLen = keyLen};
-
-      mDebug("transId:%d %s startTs:%" PRId64 "cleared due to finished", pEntry->transId, pEntry->name,
+      mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name,
              pEntry->startTime);
       taosArrayPush(pList, &info);
     } else {

source/dnode/mnode/impl/test/acct/acct.cpp

@@ -33,6 +33,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
   ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
+  rpcFreeCont(pRsp->pCont);
 }
 
 TEST_F(MndTestAcct, 02_Alter_Acct) {
@@ -43,6 +44,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
   ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
+  rpcFreeCont(pRsp->pCont);
 }
 
 TEST_F(MndTestAcct, 03_Drop_Acct) {
@@ -53,4 +55,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
   SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
   ASSERT_NE(pRsp, nullptr);
   ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
+  rpcFreeCont(pRsp->pCont);
 }

source/dnode/mnode/impl/test/sma/sma.cpp

@@ -253,12 +253,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateDbReq(dbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     pReq = BuildCreateBSmaStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
   }
@@ -269,12 +271,14 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildCreateBSmaStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     pReq = BuildDropStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 0);
   }
@@ -283,5 +287,6 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
     pReq = BuildDropStbReq(stbname, &contLen);
     pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 }

source/dnode/mnode/impl/test/stb/stb.cpp

@@ -155,6 +155,7 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -176,6 +177,7 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -204,6 +206,7 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char*
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -226,6 +229,7 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -247,6 +251,7 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -268,6 +273,7 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -290,6 +296,7 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c
   tSerializeSMAlterStbReq(pHead, contLen, &req);
 
   *pContLen = contLen;
+  taosArrayDestroy(req.pFields);
   return pHead;
 }
 
@@ -303,6 +310,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -311,6 +319,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -334,6 +343,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
 
     STableMetaRsp metaRsp = {0};
     tDeserializeSTableMetaRsp(pMsg->pCont, pMsg->contLen, &metaRsp);
+    rpcFreeCont(pMsg->pCont);
 
     EXPECT_STREQ(metaRsp.dbFName, dbname);
     EXPECT_STREQ(metaRsp.tbName, "stb");
@@ -410,6 +420,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -423,6 +434,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -436,6 +448,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -443,30 +456,35 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddTagReq("1.d3.stb", "tag4", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddTagReq("1.d2.stb3", "tag4", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddTagReq(stbname, "tag3", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddTagReq(stbname, "col1", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -474,6 +492,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
   }
@@ -483,6 +502,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -495,18 +515,21 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildCreateStbReq(stbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbDropTagReq(stbname, "tag5", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -514,6 +537,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -524,6 +548,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -536,41 +561,48 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildCreateStbReq(stbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag5", "tag6", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagNameReq(stbname, "col1", "tag6", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "col1", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
   {
     void*    pReq = BuildAlterStbUpdateTagNameReq(stbname, "tag3", "tag2", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -578,6 +610,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -588,6 +621,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -600,36 +634,42 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildCreateStbReq(stbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag5", 12, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag1", 13, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 8, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateTagBytesReq(stbname, "tag3", 20, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -640,6 +680,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -653,6 +694,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -660,30 +702,35 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddColumnReq("1.d7.stb", "tag4", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddColumnReq("1.d6.stb3", "tag4", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_STB_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddColumnReq(stbname, "tag3", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TAG_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbAddColumnReq(stbname, "col1", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_ALREADY_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -691,6 +738,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -701,6 +749,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -713,30 +762,35 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildCreateStbReq(stbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbDropColumnReq(stbname, "col4", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbDropColumnReq(stbname, "col1", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbDropColumnReq(stbname, "ts", &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_ALTER_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -744,6 +798,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
@@ -751,6 +806,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -761,6 +817,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }
 
@@ -773,42 +830,49 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildCreateDbReq(dbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildCreateStbReq(stbname, &contLen);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col5", 12, &contLen, 0);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "ts", 8, &contLen, 0);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_STB_OPTION);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 8, &contLen, 0);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", TSDB_MAX_BYTES_PER_ROW, &contLen, 0);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_ROW_BYTES);
+    rpcFreeCont(pRsp->pCont);
   }
 
   {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col1", 20, &contLen, 0);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -818,6 +882,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     void*    pReq = BuildAlterStbUpdateColumnBytesReq(stbname, "col_not_exist", 20, &contLen, 1);
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_STB, pReq, contLen);
     ASSERT_EQ(pRsp->code, TSDB_CODE_MND_COLUMN_NOT_EXIST);
+    rpcFreeCont(pRsp->pCont);
 
     test.SendShowReq(TSDB_MGMT_TABLE_STB, "ins_stables", dbname);
     EXPECT_EQ(test.GetShowRows(), 1);
@@ -828,5 +893,6 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
     SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
     ASSERT_NE(pRsp, nullptr);
     ASSERT_EQ(pRsp->code, 0);
+    rpcFreeCont(pRsp->pCont);
   }
 }

source/dnode/vnode/inc/vnode.h

@@ -154,8 +154,7 @@ typedef struct STsdbReader STsdbReader;
 #define CACHESCAN_RETRIEVE_LAST        0x8
 
 int32_t      tsdbReaderOpen2(void *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables,
-                             SSDataBlock *pResBlock, void **ppReader, const char *idstr, bool countOnly,
-                             SHashObj **pIgnoreTables);
+                             SSDataBlock *pResBlock, void **ppReader, const char *idstr, SHashObj **pIgnoreTables);
 int32_t      tsdbSetTableList2(STsdbReader *pReader, const void *pTableList, int32_t num);
 void         tsdbReaderSetId2(STsdbReader *pReader, const char *idstr);
 void         tsdbReaderClose2(STsdbReader *pReader);
@@ -170,7 +169,9 @@ void        *tsdbGetIdx2(SMeta *pMeta);
 void        *tsdbGetIvtIdx2(SMeta *pMeta);
 uint64_t     tsdbGetReaderMaxVersion2(STsdbReader *pReader);
 void         tsdbReaderSetCloseFlag(STsdbReader *pReader);
-//======================================================================================================================
+int64_t      tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
+void         tsdbSetFilesetDelimited(STsdbReader* pReader);
+void         tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 
 int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
 int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,

source/dnode/vnode/src/inc/tsdb.h

@@ -676,46 +676,57 @@ struct SDelFWriter {
 };
 
 #include "tarray2.h"
-// #include "tsdbFS2.h"
-//  struct STFileSet;
 typedef struct STFileSet STFileSet;
 typedef TARRAY2(STFileSet *) TFileSetArray;
 
-typedef struct STSnapRange STSnapRange;
-typedef TARRAY2(STSnapRange *) TSnapRangeArray;  // disjoint snap ranges
+// fset range
+typedef struct STFileSetRange STFileSetRange;
+typedef TARRAY2(STFileSetRange *) TFileSetRangeArray;  // disjoint ranges
 
-// util
-int32_t   tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
-int32_t   tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR);
-void      tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap);
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges);
-
-// snap partition list
-typedef TARRAY2(SVersionRange) SVerRangeList;
-typedef struct STsdbSnapPartition STsdbSnapPartition;
-typedef TARRAY2(STsdbSnapPartition *) STsdbSnapPartList;
-// util
-STsdbSnapPartList *tsdbSnapPartListCreate();
-void               tsdbSnapPartListDestroy(STsdbSnapPartList **ppList);
-int32_t            tSerializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tDeserializeTsdbSnapPartList(void *buf, int32_t bufLen, STsdbSnapPartList *pList);
-int32_t            tsdbSnapPartListToRangeDiff(STsdbSnapPartList *pList, TSnapRangeArray **ppRanges);
+int32_t tsdbTFileSetRangeClear(STFileSetRange **fsr);
+int32_t tsdbTFileSetRangeArrayDestroy(TFileSetRangeArray **ppArr);
 
+// fset partition
 enum {
-  TSDB_SNAP_RANGE_TYP_HEAD = 0,
-  TSDB_SNAP_RANGE_TYP_DATA,
-  TSDB_SNAP_RANGE_TYP_SMA,
-  TSDB_SNAP_RANGE_TYP_TOMB,
-  TSDB_SNAP_RANGE_TYP_STT,
-  TSDB_SNAP_RANGE_TYP_MAX,
+  TSDB_FSET_RANGE_TYP_HEAD = 0,
+  TSDB_FSET_RANGE_TYP_DATA,
+  TSDB_FSET_RANGE_TYP_SMA,
+  TSDB_FSET_RANGE_TYP_TOMB,
+  TSDB_FSET_RANGE_TYP_STT,
+  TSDB_FSET_RANGE_TYP_MAX,
 };
 
-struct STsdbSnapPartition {
+typedef TARRAY2(SVersionRange) SVerRangeList;
+
+struct STsdbFSetPartition {
   int64_t       fid;
   int8_t        stat;
-  SVerRangeList verRanges[TSDB_SNAP_RANGE_TYP_MAX];
+  SVerRangeList verRanges[TSDB_FSET_RANGE_TYP_MAX];
 };
 
+typedef struct STsdbFSetPartition STsdbFSetPartition;
+typedef TARRAY2(STsdbFSetPartition *) STsdbFSetPartList;
+
+STsdbFSetPartList *tsdbFSetPartListCreate();
+void               tsdbFSetPartListDestroy(STsdbFSetPartList **ppList);
+int32_t            tSerializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tDeserializeTsdbFSetPartList(void *buf, int32_t bufLen, STsdbFSetPartList *pList);
+int32_t            tsdbFSetPartListToRangeDiff(STsdbFSetPartList *pList, TFileSetRangeArray **ppRanges);
+
+// snap rep format
+typedef enum ETsdbRepFmt {
+  TSDB_SNAP_REP_FMT_DEFAULT = 0,
+  TSDB_SNAP_REP_FMT_RAW,
+  TSDB_SNAP_REP_FMT_HYBRID,
+} ETsdbRepFmt;
+
+typedef struct STsdbRepOpts {
+  ETsdbRepFmt format;
+} STsdbRepOpts;
+
+int32_t tSerializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
+
 // snap read
 struct STsdbReadSnap {
   SMemTable     *pMem;
@@ -773,20 +784,25 @@ typedef struct SBlockDataInfo {
   int32_t    sttBlockIndex;
 } SBlockDataInfo;
 
-typedef struct SSttBlockLoadInfo {
-  SBlockDataInfo blockData[2];      // buffered block data
-  int32_t        statisBlockIndex;  // buffered statistics block index
-  void          *statisBlock;       // buffered statistics block data
-  void          *pSttStatisBlkArray;
-  SArray        *aSttBlk;
-  int32_t        currentLoadBlockIndex;
-  STSchema      *pSchema;
-  int16_t       *colIds;
-  int32_t        numOfCols;
-  bool           checkRemainingRow;  // todo: no assign value?
-  bool           isLast;
-  bool           sttBlockLoaded;
+// todo: move away
+typedef struct {
+  SArray *pUid;
+  SArray *pFirstKey;
+  SArray *pLastKey;
+  SArray *pCount;
+} SSttTableRowsInfo;
 
+typedef struct SSttBlockLoadInfo {
+  SBlockDataInfo  blockData[2];  // buffered block data
+  SArray         *aSttBlk;
+  int32_t         currentLoadBlockIndex;
+  STSchema       *pSchema;
+  int16_t        *colIds;
+  int32_t         numOfCols;
+  bool            checkRemainingRow;  // todo: no assign value?
+  bool            isLast;
+  bool            sttBlockLoaded;
+  SSttTableRowsInfo info;
   SSttBlockLoadCostInfo cost;
 } SSttBlockLoadInfo;
 
@@ -875,27 +891,31 @@ typedef struct {
   _load_tomb_fn loadTombFn;
   void         *pReader;
   void         *idstr;
+  bool          rspRows;         // response the rows in stt-file, if possible
 } SMergeTreeConf;
 
-int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf);
+typedef struct SSttDataInfoForTable {
+  SArray* pTimeWindowList;
+  int64_t numOfRows;
+} SSttDataInfoForTable;
 
-void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
-bool tMergeTreeNext(SMergeTree *pMTree);
-void tMergeTreePinSttBlock(SMergeTree *pMTree);
-void tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
-bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
-void tMergeTreeClose(SMergeTree *pMTree);
+int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable* pTableInfo);
+void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
+bool    tMergeTreeNext(SMergeTree *pMTree);
+void    tMergeTreePinSttBlock(SMergeTree *pMTree);
+void    tMergeTreeUnpinSttBlock(SMergeTree *pMTree);
+bool    tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
+void    tMergeTreeClose(SMergeTree *pMTree);
 
 SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
-void               getSttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, SSttBlockLoadCostInfo *pLoadCost);
 void              *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
 void              *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
 
 // tsdbCache ==============================================================================================
 typedef enum {
-  READ_MODE_COUNT_ONLY = 0x1,
-  READ_MODE_ALL,
-} EReadMode;
+  READER_EXEC_DATA = 0x1,
+  READER_EXEC_ROWS = 0x2,
+} EExecMode;
 
 typedef struct {
   TSKEY   ts;
@@ -1042,7 +1062,7 @@ typedef enum {
 
 // utils
 ETsdbFsState tsdbSnapGetFsState(SVnode *pVnode);
-int32_t      tsdbSnapGetDetails(SVnode *pVnode, SSnapshot *pSnap);
+int32_t      tsdbSnapPrepDescription(SVnode *pVnode, SSnapshot *pSnap);
 
 #ifdef __cplusplus
 }

source/dnode/vnode/src/inc/vnodeInt.h

@@ -65,6 +65,8 @@ typedef struct SMetaSnapReader    SMetaSnapReader;
 typedef struct SMetaSnapWriter    SMetaSnapWriter;
 typedef struct STsdbSnapReader    STsdbSnapReader;
 typedef struct STsdbSnapWriter    STsdbSnapWriter;
+typedef struct STsdbSnapRAWReader STsdbSnapRAWReader;
+typedef struct STsdbSnapRAWWriter STsdbSnapRAWWriter;
 typedef struct STqSnapReader      STqSnapReader;
 typedef struct STqSnapWriter      STqSnapWriter;
 typedef struct STqOffsetReader    STqOffsetReader;
@@ -236,10 +238,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
 int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg);
 
-int32_t tqRestartStreamTasks(STQ* pTq);
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
 int32_t tqScanWal(STQ* pTq);
-int32_t tqStartStreamTasks(STQ* pTq);
 
 int     tqCommit(STQ*);
 int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);
@@ -313,6 +313,15 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, void* pRang
 int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, SSnapDataHdr* pHdr);
 int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter);
 int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback);
+// STsdbSnapRAWReader ========================================
+int32_t tsdbSnapRAWReaderOpen(STsdb* pTsdb, int64_t ever, int8_t type, STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWReaderClose(STsdbSnapRAWReader** ppReader);
+int32_t tsdbSnapRAWRead(STsdbSnapRAWReader* pReader, uint8_t** ppData);
+// STsdbSnapRAWWriter ========================================
+int32_t tsdbSnapRAWWriterOpen(STsdb* pTsdb, int64_t ever, STsdbSnapRAWWriter** ppWriter);
+int32_t tsdbSnapRAWWrite(STsdbSnapRAWWriter* pWriter, SSnapDataHdr* pHdr);
+int32_t tsdbSnapRAWWriterPrepareClose(STsdbSnapRAWWriter* pWriter);
+int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** ppWriter, int8_t rollback);
 // STqSnapshotReader ==
 int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader);
 int32_t tqSnapReaderClose(STqSnapReader** ppReader);
@@ -531,6 +540,7 @@ enum {
   SNAP_DATA_STREAM_STATE = 11,
   SNAP_DATA_STREAM_STATE_BACKEND = 12,
   SNAP_DATA_TQ_CHECKINFO = 13,
+  SNAP_DATA_RAW = 14,
 };
 
 struct SSnapDataHdr {

source/dnode/vnode/src/tq/tq.c

@@ -699,7 +699,23 @@ end:
   return ret;
 }
 
-void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
+static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
+
+static STaskId replaceStreamTaskId(SStreamTask* pTask) {
+  ASSERT(pTask->info.fillHistory);
+  STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
+
+  pTask->id.streamId = pTask->streamTaskId.streamId;
+  pTask->id.taskId = pTask->streamTaskId.taskId;
+
+  return id;
+}
+
+static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) {
+  ASSERT(pTask->info.fillHistory);
+  pTask->id.taskId = pId->taskId;
+  pTask->id.streamId = pId->streamId;
+}
 
 int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   int32_t vgId = TD_VID(pTq->pVnode);
@@ -713,15 +729,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
   streamTaskOpenAllUpstreamInput(pTask);
 
   if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
-    SStreamTask* pStateTask = pTask;
-
-    STaskId taskId = {.streamId = 0, .taskId = 0};
+    STaskId taskId = {0};
     if (pTask->info.fillHistory) {
-      taskId.streamId = pTask->id.streamId;
-      taskId.taskId = pTask->id.taskId;
-
-      pTask->id.streamId = pTask->streamTaskId.streamId;
-      pTask->id.taskId = pTask->streamTaskId.taskId;
+      taskId = replaceStreamTaskId(pTask);
     }
 
     pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
@@ -731,9 +741,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     } else {
       tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
     }
+
     if (pTask->info.fillHistory) {
-      pTask->id.streamId = taskId.streamId;
-      pTask->id.taskId = taskId.taskId;
+      restoreStreamTaskId(pTask, &taskId);
     }
 
     SReadHandle handle = {
@@ -754,15 +764,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
 
     qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId);
   } else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
-    SStreamTask* pSateTask = pTask;
-    // SStreamTask task = {0};
-
-    STaskId taskId = {.streamId = 0, .taskId = 0};
+    STaskId taskId = {0};
     if (pTask->info.fillHistory) {
-      taskId.streamId = pTask->id.streamId;
-      taskId.taskId = pTask->id.taskId;
-      pTask->id.streamId = pTask->streamTaskId.streamId;
-      pTask->id.taskId = pTask->streamTaskId.taskId;
+      taskId = replaceStreamTaskId(pTask);
     }
 
     pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
@@ -774,15 +778,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     }
 
     if (pTask->info.fillHistory) {
-      pTask->id.streamId = taskId.streamId;
-      pTask->id.taskId = taskId.taskId;
+      restoreStreamTaskId(pTask, &taskId);
     }
 
-    int32_t     numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
     SReadHandle handle = {
         .checkpointId = pTask->chkInfo.checkpointId,
         .vnode = NULL,
-        .numOfVgroups = numOfVgroups,
+        .numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList),
         .pStateBackend = pTask->pState,
         .fillHistory = pTask->info.fillHistory,
         .winRange = pTask->dataRange.window,
@@ -828,12 +830,6 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
     pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
   }
 
-  //  // reset the task status from unfinished transaction
-  //  if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
-  //    tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
-  //    pTask->status.taskStatus = TASK_STATUS__READY;
-  //  }
-
   streamTaskResetUpstreamStageInfo(pTask);
   streamSetupScheduleTrigger(pTask);
   SCheckpointInfo* pChkInfo = &pTask->chkInfo;

source/dnode/vnode/src/tq/tqStreamStateSnap.c

@@ -181,5 +181,5 @@ int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId)
 }
 
 int32_t streamStateLoadTasks(SStreamStateWriter* pWriter) {
-  return streamMetaReloadAllTasks(pWriter->pTq->pStreamMeta);
+  return streamMetaLoadAllTasks(pWriter->pTq->pStreamMeta);
 }

source/dnode/vnode/src/tq/tqUtil.c

@@ -168,6 +168,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
     taosWUnLockLatch(&pTq->lock);
   }
 
+  dataRsp.reqOffset = *pOffset;   // reqOffset represents the current date offset, may be changed if wal not exists
   code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId);
 
 end : {

source/dnode/vnode/src/tqCommon/tqCommon.c

@@ -712,9 +712,9 @@ int32_t resetStreamTaskStatus(SStreamMeta* pMeta) {
 }
 
 static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
-  int32_t      vgId = pMeta->vgId;
-  int32_t      code = 0;
-  int64_t      st = taosGetTimestampMs();
+  int32_t vgId = pMeta->vgId;
+  int32_t code = 0;
+  int64_t st = taosGetTimestampMs();
 
   while(1) {
     int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1);
@@ -736,17 +736,10 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
   }
 
   streamMetaWLock(pMeta);
-  // code = streamMetaReopen(pMeta);
-  // if (code != TSDB_CODE_SUCCESS) {
-  //   tqError("vgId:%d failed to reopen stream meta", vgId);
-  //   streamMetaWUnLock(pMeta);
-  //   code = terrno;
-  //   return code;
-  // }
+  
+  streamMetaClear(pMeta);
 
-  // streamMetaInitBackend(pMeta);
   int64_t el = taosGetTimestampMs() - st;
-
   tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.);
 
   code = streamMetaLoadAllTasks(pMeta);
@@ -758,10 +751,10 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) {
   }
 
   if (isLeader && !tsDisableStream) {
-    tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
     resetStreamTaskStatus(pMeta);
-
     streamMetaWUnLock(pMeta);
+    tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
+
     startStreamTasks(pMeta);
   } else {
     streamMetaResetStartInfo(&pMeta->startInfo);

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -53,11 +53,10 @@ static void tsdbCloseBICache(STsdb *pTsdb) {
 }
 
 static int32_t tsdbOpenBCache(STsdb *pTsdb) {
-  int32_t code = 0;
-  // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5);
-  int32_t szPage = pTsdb->pVnode->config.tsdbPageSize;
-
-  SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * tsS3BlockSize * szPage, 0, .5);
+  int32_t    code = 0;
+  int32_t    szPage = pTsdb->pVnode->config.tsdbPageSize;
+  int64_t    szBlock = tsS3BlockSize <= 1024 ? 1024 : tsS3BlockSize;
+  SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * szBlock * szPage, 0, .5);
   if (pCache == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -67,8 +66,9 @@ static int32_t tsdbOpenBCache(STsdb *pTsdb) {
 
   taosThreadMutexInit(&pTsdb->bMutex, NULL);
 
-_err:
   pTsdb->bCache = pCache;
+
+_err:
   return code;
 }
 
@@ -452,9 +452,11 @@ static SLastCol *tsdbCacheLookup(STsdb *pTsdb, tb_uid_t uid, int16_t cid, int8_t
 static void reallocVarData(SColVal *pColVal) {
   if (IS_VAR_DATA_TYPE(pColVal->type)) {
     uint8_t *pVal = pColVal->value.pData;
-    pColVal->value.pData = taosMemoryMalloc(pColVal->value.nData);
-    if (pColVal->value.nData) {
+    if (pColVal->value.nData > 0) {
+      pColVal->value.pData = taosMemoryMalloc(pColVal->value.nData);
       memcpy(pColVal->value.pData, pVal, pColVal->value.nData);
+    } else {
+      pColVal->value.pData = NULL;
     }
   }
 }
@@ -1167,26 +1169,33 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
 
     // taosThreadMutexLock(&pTsdb->lruMutex);
 
+    bool       erase = false;
     LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen);
     if (h) {
       SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h);
-      if (pLastCol->dirty) {
+      if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
         pLastCol->dirty = 0;
+        erase = true;
       }
-      taosLRUCacheRelease(pTsdb->lruCache, h, true);
+      taosLRUCacheRelease(pTsdb->lruCache, h, erase);
+    }
+    if (erase) {
+      taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
     }
-    taosLRUCacheErase(pTsdb->lruCache, keys_list[i], klen);
 
+    erase = false;
     h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen);
     if (h) {
       SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h);
-      if (pLastCol->dirty) {
+      if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) {
         pLastCol->dirty = 0;
+        erase = true;
       }
-      taosLRUCacheRelease(pTsdb->lruCache, h, true);
+      taosLRUCacheRelease(pTsdb->lruCache, h, erase);
+    }
+    if (erase) {
+      taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
     }
-    taosLRUCacheErase(pTsdb->lruCache, keys_list[num_keys + i], klen);
-
     // taosThreadMutexUnlock(&pTsdb->lruMutex);
   }
   for (int i = 0; i < num_keys; ++i) {
@@ -1797,8 +1806,8 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
       }
 
       if (record.version <= pReader->info.verRange.maxVer) {
-        tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
-                  TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
+        /*tsdbError("tomb xx load/cache: vgId:%d fid:%d record %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
+          TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);*/
 
         SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
         taosArrayPush(pInfo->pTombData, &delData);
@@ -1870,7 +1879,7 @@ static int32_t lastIterOpen(SFSLastIter *iter, STFileSet *pFileSet, STsdb *pTsdb
       .idstr = pr->idstr,
   };
 
-  code = tMergeTreeOpen2(&iter->mergeTree, &conf);
+  code = tMergeTreeOpen2(&iter->mergeTree, &conf, NULL);
   if (code != TSDB_CODE_SUCCESS) {
     return -1;
   }
@@ -2748,14 +2757,16 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
 
         *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
         if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
-          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-          if (pCol->colVal.value.pData == NULL) {
-            terrno = TSDB_CODE_OUT_OF_MEMORY;
-            code = TSDB_CODE_OUT_OF_MEMORY;
-            goto _err;
-          }
           if (pColVal->value.nData > 0) {
+            pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
+            if (pCol->colVal.value.pData == NULL) {
+              terrno = TSDB_CODE_OUT_OF_MEMORY;
+              code = TSDB_CODE_OUT_OF_MEMORY;
+              goto _err;
+            }
             memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          } else {
+            pCol->colVal.value.pData = NULL;
           }
         }
 
@@ -2795,17 +2806,21 @@ static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SC
       tsdbRowGetColVal(pRow, pTSchema, slotIds[iCol], pColVal);
       if (!COL_VAL_IS_VALUE(tColVal) && COL_VAL_IS_VALUE(pColVal)) {
         SLastCol lastCol = {.ts = rowTs, .colVal = *pColVal};
-        if (IS_VAR_DATA_TYPE(pColVal->type) && pColVal->value.nData > 0) {
+        if (IS_VAR_DATA_TYPE(pColVal->type) /* && pColVal->value.nData > 0 */) {
           SLastCol *pLastCol = (SLastCol *)taosArrayGet(pColArray, iCol);
           taosMemoryFree(pLastCol->colVal.value.pData);
 
-          lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
-          if (lastCol.colVal.value.pData == NULL) {
-            terrno = TSDB_CODE_OUT_OF_MEMORY;
-            code = TSDB_CODE_OUT_OF_MEMORY;
-            goto _err;
+          if (pColVal->value.nData > 0) {
+            lastCol.colVal.value.pData = taosMemoryMalloc(lastCol.colVal.value.nData);
+            if (lastCol.colVal.value.pData == NULL) {
+              terrno = TSDB_CODE_OUT_OF_MEMORY;
+              code = TSDB_CODE_OUT_OF_MEMORY;
+              goto _err;
+            }
+            memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          } else {
+            lastCol.colVal.value.pData = NULL;
           }
-          memcpy(lastCol.colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
         }
 
         taosArraySet(pColArray, iCol, &lastCol);
@@ -2916,14 +2931,18 @@ static int32_t mergeLastRowCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray,
 
       *pCol = (SLastCol){.ts = rowTs, .colVal = *pColVal};
       if (IS_VAR_DATA_TYPE(pColVal->type) /*&& pColVal->value.nData > 0*/) {
-        pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
-        if (pCol->colVal.value.pData == NULL) {
-          terrno = TSDB_CODE_OUT_OF_MEMORY;
-          code = TSDB_CODE_OUT_OF_MEMORY;
-          goto _err;
-        }
         if (pColVal->value.nData > 0) {
-          memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          pCol->colVal.value.pData = taosMemoryMalloc(pCol->colVal.value.nData);
+          if (pCol->colVal.value.pData == NULL) {
+            terrno = TSDB_CODE_OUT_OF_MEMORY;
+            code = TSDB_CODE_OUT_OF_MEMORY;
+            goto _err;
+          }
+          if (pColVal->value.nData > 0) {
+            memcpy(pCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
+          }
+        } else {
+          pCol->colVal.value.pData = NULL;
         }
       }
 

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -22,19 +22,34 @@
 
 #define HASTYPE(_type, _t) (((_type) & (_t)) == (_t))
 
+static void setFirstLastResColToNull(SColumnInfoData* pCol, int32_t row) {
+  char *buf = taosMemoryCalloc(1, pCol->info.bytes);
+  SFirstLastRes* pRes = (SFirstLastRes*)((char*)buf + VARSTR_HEADER_SIZE);
+  pRes->bytes = 0;
+  pRes->hasResult = true;
+  pRes->isNull = true;
+  varDataSetLen(buf, pCol->info.bytes - VARSTR_HEADER_SIZE);
+  colDataSetVal(pCol, row, buf, false);
+  taosMemoryFree(buf);
+}
+
 static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* pReader, const int32_t* slotIds,
                           const int32_t* dstSlotIds, void** pRes, const char* idStr) {
   int32_t numOfRows = pBlock->info.rows;
   // bool    allNullRow = true;
 
   if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) {
-    uint64_t ts = 0;
-    SFirstLastRes* p;
-    col_id_t colId;
+    uint64_t ts = TSKEY_MIN;
+    SFirstLastRes* p = NULL;
+    col_id_t colId = -1;
     for (int32_t i = 0; i < pReader->numOfCols; ++i) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
-      int32_t          slotId = slotIds[i];
-      SLastCol*        pColVal = (SLastCol*)taosArrayGet(pRow, i);
+      if (slotIds[i] == -1) {
+        setFirstLastResColToNull(pColInfoData, numOfRows);
+        continue;
+      }
+      int32_t   slotId = slotIds[i];
+      SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       colId = pColVal->colVal.cid;
       p = (SFirstLastRes*)varDataVal(pRes[i]);
 
@@ -63,10 +78,14 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
     for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) {
       SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx);
       if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
-        colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        if (ts == TSKEY_MIN) {
+          colDataSetNULL(pCol, numOfRows);
+        } else {
+          colDataSetVal(pCol, numOfRows, (const char*)&ts, false);
+        }
         continue;
-      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) {
-        if (!p->isNull) {
+      } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && (pCol->info.colId == colId || colId == -1)) {
+        if (p && !p->isNull) {
           colDataSetVal(pCol, numOfRows, p->buf, false);
         } else {
           colDataSetNULL(pCol, numOfRows);
@@ -81,6 +100,10 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]);
 
       int32_t   slotId = slotIds[i];
+      if (slotId == -1) {
+        colDataSetNULL(pColInfoData, numOfRows);
+        continue;
+      }
       SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i);
       SColVal*  pVal = &pColVal->colVal;
 
@@ -300,7 +323,13 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
   }
 
   for (int32_t j = 0; j < pr->numOfCols; ++j) {
-    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + pr->pSchema->columns[slotIds[j]].bytes + VARSTR_HEADER_SIZE);
+    int32_t bytes;
+    if (slotIds[j] == -1)
+      bytes = 1;
+    else
+      bytes = pr->pSchema->columns[slotIds[j]].bytes;
+
+    pRes[j] = taosMemoryCalloc(1, sizeof(SFirstLastRes) + bytes + VARSTR_HEADER_SIZE);
     SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[j]);
     p->ts = INT64_MIN;
   }
@@ -324,6 +353,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
 
     for (int32_t i = 0; i < pr->numOfCols; ++i) {
       int32_t          slotId = slotIds[i];
+      if (slotId == -1) {
+        SLastCol p = {.ts = INT64_MIN, .colVal.type = TSDB_DATA_TYPE_BOOL, .colVal.flag = CV_FLAG_NULL};
+        taosArrayPush(pLastCols, &p);
+        continue;
+      }
       struct STColumn* pCol = &pr->pSchema->columns[slotId];
       SLastCol         p = {.ts = INT64_MIN, .colVal.type = pCol->type, .colVal.flag = CV_FLAG_NULL};
 
@@ -348,6 +382,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32
         bool    hasNotNullRow = true;
         int64_t singleTableLastTs = INT64_MAX;
         for (int32_t k = 0; k < pr->numOfCols; ++k) {
+          if (slotIds[k] == -1) continue;
           SLastCol* p = taosArrayGet(pLastCols, k);
           SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, k);
 

source/dnode/vnode/src/tsdb/tsdbCommit2.c

@@ -435,7 +435,7 @@ _exit:
     tsdbDebug("vgId:%d %s done, fid:%d minKey:%" PRId64 " maxKey:%" PRId64 " expLevel:%d", TD_VID(tsdb->pVnode),
               __func__, committer->ctx->fid, committer->ctx->minKey, committer->ctx->maxKey, committer->ctx->expLevel);
   }
-  return 0;
+  return code;
 }
 
 static int32_t tsdbCommitFileSetEnd(SCommitter2 *committer) {

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.c

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbDataFileRAW.h"
+
+// SDataFileRAWReader =============================================
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  reader[0] = taosMemoryCalloc(1, sizeof(SDataFileRAWReader));
+  if (reader[0] == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  reader[0]->config[0] = config[0];
+
+  if (fname) {
+    if (fname) {
+      code = tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else {
+    char fname1[TSDB_FILENAME_LEN];
+    tsdbTFileName(config->tsdb, &config->file, fname1);
+    code = tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader) {
+  if (reader[0] == NULL) return 0;
+
+  if (reader[0]->fd) {
+    tsdbCloseFile(&reader[0]->fd);
+  }
+
+  taosMemoryFree(reader[0]);
+  reader[0] = NULL;
+  return 0;
+}
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *pBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  pBlock->file.type = reader->config->file.type;
+  pBlock->file.fid = reader->config->file.fid;
+  pBlock->file.cid = reader->config->file.cid;
+  pBlock->file.size = reader->config->file.size;
+  pBlock->file.minVer = reader->config->file.minVer;
+  pBlock->file.maxVer = reader->config->file.maxVer;
+  pBlock->file.stt->level = reader->config->file.stt->level;
+
+  code = tsdbReadFile(reader->fd, pBlock->offset, pBlock->data, pBlock->dataLength, 0);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(reader->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+// SDataFileRAWWriter =============================================
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **ppWriter) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  SDataFileRAWWriter *writer = taosMemoryCalloc(1, sizeof(SDataFileRAWWriter));
+  if (!writer) return TSDB_CODE_OUT_OF_MEMORY;
+
+  writer->config[0] = config[0];
+
+  code = tsdbDataFileRAWWriterDoOpen(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    taosMemoryFree(writer);
+    writer = NULL;
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  ppWriter[0] = writer;
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterCloseAbort(SDataFileRAWWriter *writer) {
+  ASSERT(0);
+  return 0;
+}
+
+static int32_t tsdbDataFileRAWWriterDoClose(SDataFileRAWWriter *writer) { return 0; }
+
+static int32_t tsdbDataFileRAWWriterCloseCommit(SDataFileRAWWriter *writer, TFileOpArray *opArr) {
+  int32_t  code = 0;
+  int32_t  lino = 0;
+  ASSERT(writer->ctx->offset <= writer->file.size);
+  ASSERT(writer->config->fid == writer->file.fid);
+
+  STFileOp op = (STFileOp){
+      .optype = TSDB_FOP_CREATE,
+      .fid = writer->config->fid,
+      .nf = writer->file,
+  };
+  code = TARRAY2_APPEND(opArr, op);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  if (writer->fd) {
+    code = tsdbFsyncFile(writer->fd);
+    TSDB_CHECK_CODE(code, lino, _exit);
+    tsdbCloseFile(&writer->fd);
+  }
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+static int32_t tsdbDataFileRAWWriterOpenDataFD(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  char    fname[TSDB_FILENAME_LEN];
+  int32_t flag = TD_FILE_READ | TD_FILE_WRITE;
+
+  if (writer->ctx->offset == 0) {
+    flag |= (TD_FILE_CREATE | TD_FILE_TRUNC);
+  }
+
+  tsdbTFileName(writer->config->tsdb, &writer->file, fname);
+  code = tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  writer->file = writer->config->file;
+  writer->ctx->offset = 0;
+
+  code = tsdbDataFileRAWWriterOpenDataFD(writer);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->opened = true;
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr) {
+  if (writer[0] == NULL) return 0;
+
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  if (writer[0]->ctx->opened) {
+    if (abort) {
+      code = tsdbDataFileRAWWriterCloseAbort(writer[0]);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    } else {
+      code = tsdbDataFileRAWWriterCloseCommit(writer[0], opArr);
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    tsdbDataFileRAWWriterDoClose(writer[0]);
+  }
+  taosMemoryFree(writer[0]);
+  writer[0] = NULL;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer[0]->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}
+
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *pDataBlock) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  code = tsdbWriteFile(writer->fd, writer->ctx->offset, (const uint8_t *)pDataBlock->data, pDataBlock->dataLength);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+  writer->ctx->offset += pDataBlock->dataLength;
+
+_exit:
+  if (code) {
+    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
+  }
+  return code;
+}

source/dnode/vnode/src/tsdb/tsdbDataFileRAW.h

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tarray2.h"
+#include "tsdbDef.h"
+#include "tsdbFSet2.h"
+#include "tsdbFile2.h"
+#include "tsdbUtil2.h"
+
+#ifndef _TSDB_DATA_FILE_RAW_H
+#define _TSDB_DATA_FILE_RAW_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// STsdbDataRAWBlockHeader =======================================
+typedef struct STsdbDataRAWBlockHeader {
+  struct {
+    int32_t type;
+    int64_t fid;
+    int64_t cid;
+    int64_t size;
+    int64_t minVer;
+    int64_t maxVer;
+    union {
+      struct {
+        int32_t level;
+      } stt[1];
+    };
+  } file;
+
+  int64_t offset;
+  int64_t dataLength;
+  uint8_t data[0];
+} STsdbDataRAWBlockHeader;
+
+// SDataFileRAWReader =============================================
+typedef struct SDataFileRAWReaderConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  STFile file;
+} SDataFileRAWReaderConfig;
+
+typedef struct SDataFileRAWReader {
+  SDataFileRAWReaderConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STsdbFD *fd;
+} SDataFileRAWReader;
+
+typedef TARRAY2(SDataFileRAWReader *) SDataFileRAWReaderArray;
+
+int32_t tsdbDataFileRAWReaderOpen(const char *fname, const SDataFileRAWReaderConfig *config,
+                                  SDataFileRAWReader **reader);
+int32_t tsdbDataFileRAWReaderClose(SDataFileRAWReader **reader);
+
+int32_t tsdbDataFileRAWReadBlockData(SDataFileRAWReader *reader, STsdbDataRAWBlockHeader *bHdr);
+
+// SDataFileRAWWriter =============================================
+typedef struct SDataFileRAWWriterConfig {
+  STsdb  *tsdb;
+  int32_t szPage;
+
+  SDiskID did;
+  int64_t fid;
+  int64_t cid;
+  int32_t level;
+
+  STFile file;
+} SDataFileRAWWriterConfig;
+
+typedef struct SDataFileRAWWriter {
+  SDataFileRAWWriterConfig config[1];
+
+  struct {
+    bool    opened;
+    int64_t offset;
+  } ctx[1];
+
+  STFile   file;
+  STsdbFD *fd;
+} SDataFileRAWWriter;
+
+typedef struct SDataFileRAWWriter SDataFileRAWWriter;
+
+int32_t tsdbDataFileRAWWriterOpen(const SDataFileRAWWriterConfig *config, SDataFileRAWWriter **writer);
+int32_t tsdbDataFileRAWWriterClose(SDataFileRAWWriter **writer, bool abort, TFileOpArray *opArr);
+
+int32_t tsdbDataFileRAWWriterDoOpen(SDataFileRAWWriter *writer);
+int32_t tsdbDataFileRAWWriteBlockData(SDataFileRAWWriter *writer, const STsdbDataRAWBlockHeader *bHdr);
+int32_t tsdbDataFileRAWFlush(SDataFileRAWWriter *writer);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TSDB_DATA_FILE_RAW_H*/

source/dnode/vnode/src/tsdb/tsdbFS2.c

@@ -529,7 +529,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) {
     for (const STfsFile *file = NULL; (file = tfsReaddir(dir)) != NULL;) {
       if (taosIsDir(file->aname)) continue;
 
-      if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) {
+      if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL &&
+          strncmp(file->aname + strlen(file->aname) - 3, ".cp", 3)) {
         int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs);
         remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1);
       }
@@ -1072,7 +1073,25 @@ int32_t tsdbFSDestroyRefSnapshot(TFileSetArray **fsetArr) {
   return 0;
 }
 
-int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRanges, TFileSetArray **fsetArr,
+static SHashObj *tsdbFSetRangeArrayToHash(TFileSetRangeArray *pRanges) {
+  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
+  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
+  if (pHash == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
+    STFileSetRange *u = TARRAY2_GET(pRanges, i);
+    int32_t         fid = u->fid;
+    int32_t         code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
+    ASSERT(code == 0);
+    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
+  }
+  return pHash;
+}
+
+int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRanges, TFileSetArray **fsetArr,
                                        TFileOpArray *fopArr) {
   int32_t    code = 0;
   STFileSet *fset;
@@ -1084,7 +1103,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
   TARRAY2_INIT(fsetArr[0]);
 
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1095,8 +1114,8 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TSnapRangeArray *pRange
   TARRAY2_FOREACH(fs->fSetArr, fset) {
     int64_t ever = VERSION_MAX;
     if (pHash) {
-      int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      int32_t         fid = fset->fid;
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         ever = u->sver - 1;
       }
@@ -1123,30 +1142,14 @@ _out:
   return code;
 }
 
-SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges) {
-  int32_t   capacity = TARRAY2_SIZE(pRanges) * 2;
-  SHashObj *pHash = taosHashInit(capacity, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK);
-  if (pHash == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
-  }
+int32_t tsdbFSDestroyCopyRangedSnapshot(TFileSetArray **fsetArr) { return tsdbFSDestroyCopySnapshot(fsetArr); }
 
-  for (int32_t i = 0; i < TARRAY2_SIZE(pRanges); i++) {
-    STSnapRange *u = TARRAY2_GET(pRanges, i);
-    int32_t      fid = u->fid;
-    int32_t      code = taosHashPut(pHash, &fid, sizeof(fid), u, sizeof(*u));
-    ASSERT(code == 0);
-    tsdbDebug("range diff hash fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
-  }
-  return pHash;
-}
-
-int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TSnapRangeArray *pRanges,
-                                      TSnapRangeArray **fsrArr) {
-  int32_t      code = 0;
-  STFileSet   *fset;
-  STSnapRange *fsr1 = NULL;
-  SHashObj    *pHash = NULL;
+int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ever, TFileSetRangeArray *pRanges,
+                                      TFileSetRangeArray **fsrArr) {
+  int32_t         code = 0;
+  STFileSet      *fset;
+  STFileSetRange *fsr1 = NULL;
+  SHashObj       *pHash = NULL;
 
   fsrArr[0] = taosMemoryCalloc(1, sizeof(*fsrArr[0]));
   if (fsrArr[0] == NULL) {
@@ -1156,7 +1159,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
   tsdbInfo("pRanges size:%d", (pRanges == NULL ? 0 : TARRAY2_SIZE(pRanges)));
   if (pRanges) {
-    pHash = tsdbGetSnapRangeHash(pRanges);
+    pHash = tsdbFSetRangeArrayToHash(pRanges);
     if (pHash == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _out;
@@ -1169,8 +1172,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
     int64_t ever1 = ever;
 
     if (pHash) {
-      int32_t      fid = fset->fid;
-      STSnapRange *u = taosHashGet(pHash, &fid, sizeof(fid));
+      int32_t         fid = fset->fid;
+      STFileSetRange *u = taosHashGet(pHash, &fid, sizeof(fid));
       if (u) {
         sver1 = u->sver;
         tsdbDebug("range hash get fid:%d, sver:%" PRId64 ", ever:%" PRId64, u->fid, u->sver, u->ever);
@@ -1184,7 +1187,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
 
     tsdbDebug("fsrArr:%p, fid:%d, sver:%" PRId64 ", ever:%" PRId64, fsrArr, fset->fid, sver1, ever1);
 
-    code = tsdbTSnapRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
+    code = tsdbTFileSetRangeInitRef(fs->tsdb, fset, sver1, ever1, &fsr1);
     if (code) break;
 
     code = TARRAY2_APPEND(fsrArr[0], fsr1);
@@ -1195,8 +1198,8 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
   taosThreadMutexUnlock(&fs->tsdb->mutex);
 
   if (code) {
-    tsdbTSnapRangeClear(&fsr1);
-    TARRAY2_DESTROY(fsrArr[0], tsdbTSnapRangeClear);
+    tsdbTFileSetRangeClear(&fsr1);
+    TARRAY2_DESTROY(fsrArr[0], tsdbTFileSetRangeClear);
     fsrArr[0] = NULL;
   }
 
@@ -1206,4 +1209,6 @@ _out:
     pHash = NULL;
   }
   return code;
-}
+}
+
+int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr) { return tsdbTFileSetRangeArrayDestroy(fsrArr); }