other: merge main into 3.0

.gitignore

@@ -11,6 +11,7 @@ CMakeSettings.json
 cmake-build-debug/
 cmake-build-release/
 cscope.out
+cscope.files
 .DS_Store
 debug/
 release/

CMakeLists.txt

@@ -41,6 +41,7 @@ add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(utils)
 add_subdirectory(examples/c)
+add_subdirectory(tests)
 include(${TD_SUPPORT_DIR}/cmake.install)
 
 # docs 

Jenkinsfile2

@@ -393,7 +393,7 @@ pipeline {
                     agent{label " Mac_catalina "}
                     steps {
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 20, unit: 'MINUTES'){
+                            timeout(time: 30, unit: 'MINUTES'){
                                 pre_test()
                                 pre_test_build_mac()
                             }

cmake/cmake.define

@@ -97,7 +97,15 @@ ENDIF()
 SET(JEMALLOC_ENABLED OFF)
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
+    IF (${CMAKE_BUILD_TYPE} MATCHES "Release")
+      MESSAGE("${Green} will build Release version! ${ColourReset}")
+      SET(COMMON_FLAGS "/W3 /D_WIN32 /DWIN32 /Zi- /O2 /GL /MD")
+
+    ELSE ()
+      MESSAGE("${Green} will build Debug version! ${ColourReset}")
+      SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
+    ENDIF()
+
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
     # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
     #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
@@ -173,17 +181,17 @@ ELSE ()
         ENDIF()
         MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
 
-        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
-            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
-        ENDIF()
-
-        IF (COMPILER_SUPPORT_AVX512VL)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
-        MESSAGE(STATUS "avx512vl supported by compiler")
-        ENDIF()
+#        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
+#            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
+#        ENDIF()
+#
+#        IF (COMPILER_SUPPORT_AVX512VL)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
+#        MESSAGE(STATUS "avx512vl supported by compiler")
+#        ENDIF()
     ENDIF()
 
     # build mode

cmake/cmake.options

@@ -151,6 +151,7 @@ IF(${BUILD_S3})
 
 IF(${BUILD_WITH_S3})
 
+add_definitions(-DUSE_S3)
 option(BUILD_WITH_COS "If build with cos" OFF)
 
 ELSE ()

contrib/CMakeLists.txt

@@ -284,7 +284,8 @@ if (${BUILD_WITH_ROCKSDB})
                 SET(CMAKE_BUILD_TYPE Release)
             endif()
         endif(${TD_LINUX})
-        MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+        MESSAGE(STATUS "ROCKSDB CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+        MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
 
         if(${TD_DARWIN})
             SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
@@ -296,8 +297,12 @@ if (${BUILD_WITH_ROCKSDB})
 
         if (${TD_WINDOWS})
             SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819")
             option(WITH_JNI "" OFF)
-            option(WITH_MD_LIBRARY "build with MD" OFF)
+            if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd")
+               message("Rocksdb build runtime lib use /MT or /MTd")
+               option(WITH_MD_LIBRARY "build with MD" OFF)
+            endif()
             set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
         endif(${TD_WINDOWS})
 
@@ -328,9 +333,11 @@ if (${BUILD_WITH_ROCKSDB})
         )
     else() 
         if (NOT ${TD_LINUX})
-            MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+            MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+            MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
             if(${TD_DARWIN})
                 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
+                SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=maybe-uninitialized")
             endif(${TD_DARWIN})
 
             if (${TD_DARWIN_ARM64})
@@ -339,8 +346,12 @@ if (${BUILD_WITH_ROCKSDB})
 
             if (${TD_WINDOWS})
                 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+                SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819")
                 option(WITH_JNI "" OFF)
-                option(WITH_MD_LIBRARY "build with MD" OFF)
+                if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd")
+                    message("Rocksdb build runtime lib use /MT or /MTd")
+                    option(WITH_MD_LIBRARY "build with MD" OFF)
+                endif()
                 set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
             endif(${TD_WINDOWS})
 
@@ -406,7 +417,9 @@ endif()
 
 # pthread
 if(${BUILD_PTHREAD})
-    set(CMAKE_BUILD_TYPE debug)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+    SET(CMAKE_BUILD_TYPE Release)
+    endif()
     add_definitions(-DPTW32_STATIC_LIB)
     add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
@@ -554,13 +567,18 @@ if(${BUILD_GEOS})
     if(${TD_LINUX})
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}")
-        IF ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+        if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
             SET(CMAKE_BUILD_TYPE Release)
         endif()
     endif(${TD_LINUX})
     option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF)
     add_subdirectory(geos EXCLUDE_FROM_ALL)
-    unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
+    if (${TD_WINDOWS})
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+    else ()
+        unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
+    endif(${TD_WINDOWS})
     target_include_directories(
         geos_c
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/geos/include>

docs/en/07-develop/01-connect/index.md

@@ -177,7 +177,7 @@ Just need to add the reference to [TDengine.Connector](https://www.nuget.org/pac
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.0" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.0" />
   </ItemGroup>
 
 </Project>

docs/en/07-develop/04-query-data/_cs_async.mdx

@@ -1,3 +0,0 @@
-```csharp
-{{#include docs/examples/csharp/asyncQuery/Program.cs}}
-```

docs/en/07-develop/04-query-data/index.mdx

@@ -15,7 +15,6 @@ import CQuery from "./_c.mdx";
 import PhpQuery from "./_php.mdx";
 import PyAsync from "./_py_async.mdx";
 import NodeAsync from "./_js_async.mdx";
-import CsAsync from "./_cs_async.mdx";
 import CAsync from "./_c_async.mdx";
 
 ## Introduction
@@ -174,9 +173,6 @@ Please note that async query can only be used with a native connection.
   <TabItem label="Python" value="python">
     <PyAsync />
   </TabItem>
-  <TabItem label="C#" value="csharp">
-    <CsAsync />
-  </TabItem>
   <TabItem label="C" value="c">
     <CAsync />
   </TabItem>

docs/en/07-develop/07-tmq.mdx

@@ -249,23 +249,23 @@ function close()
 <TabItem value="C#" label="C#">
 
 ```csharp
+class ConsumerBuilder<TValue>
+
 ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
 
-virtual IConsumer Build()
-
-Consumer(ConsumerBuilder builder)
+public IConsumer<TValue> Build()
 
 void Subscribe(IEnumerable<string> topics)
 
 void Subscribe(string topic) 
 
-ConsumeResult Consume(int millisecondsTimeout)
+ConsumeResult<TValue> Consume(int millisecondsTimeout)
 
 List<string> Subscription()
 
 void Unsubscribe()
  
-void Commit(ConsumeResult consumerResult)
+List<TopicPartitionOffset> Commit()
 
 void Close()
 ```
@@ -501,25 +501,19 @@ let consumer = taos.consumer({
 <TabItem value="C#" label="C#">
 
 ```csharp
-using TDengineTMQ;
-
-// Create consumer groups on demand (GourpID) and enable automatic commits (EnableAutoCommit),
-// an automatic commit interval (AutoCommitIntervalMs), and a username (TDConnectUser) and password (TDConnectPasswd)
-var cfg = new ConsumerConfig
- {
-    EnableAutoCommit = "true"
-    AutoCommitIntervalMs = "1000"
-    GourpId = "TDengine-TMQ-C#",
-    TDConnectUser = "root",
-    TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "latest"
-    MsgWithTableName = "true",
-    TDConnectIp = "127.0.0.1",
-    TDConnectPort = "6030"
- };
-
-var consumer = new ConsumerBuilder(cfg).Build();
-
+var cfg = new Dictionary<string, string>()
+{
+    { "group.id", "group1" },
+    { "auto.offset.reset", "latest" },
+    { "td.connect.ip", "127.0.0.1" },
+    { "td.connect.user", "root" },
+    { "td.connect.pass", "taosdata" },
+    { "td.connect.port", "6030" },
+    { "client.id", "tmq_example" },
+    { "enable.auto.commit", "true" },
+    { "msg.with.table.name", "false" },
+};
+var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
 ```
 
 </TabItem>
@@ -748,10 +742,12 @@ while(true){
 ## Consume data
 while (true)
 {
-    var consumerRes = consumer.Consume(100);
-    // process ConsumeResult
-    ProcessMsg(consumerRes);
-    consumer.Commit(consumerRes);
+    using (var result = consumer.Consume(500))
+    {
+        if (result == null) continue;
+        ProcessMsg(result);
+        consumer.Commit();
+    }
 }
 ```
 

docs/en/08-client-libraries/04-java.mdx

@@ -148,7 +148,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the
 
 **Note**: Only TAG supports JSON types
 Due to historical reasons, the BINARY type data in TDengine is not truly binary data and is no longer recommended for use. Please use VARBINARY type instead.  
-GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type](../../taos-sql/data-type/)  
+GEOMETRY type is binary data in little endian byte order, which complies with the WKB specification. For detailed information, please refer to [Data Type](../../taos-sql/data-type/)
 For WKB specifications, please refer to [Well Known Binary (WKB)](https://libgeos.org/specifications/wkb/)
 For Java connector, the jts library can be used to easily create GEOMETRY type objects, serialize them, and write them to TDengine. Here is an example [Geometry example](https://github.com/taosdata/TDengine/blob/3.0/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/GeometryDemo.java)
 

docs/en/08-client-libraries/05-go.mdx

@@ -74,7 +74,7 @@ If it is a TDengine error, you can get the error code and error information in t
 ### Pre-installation preparation
 
 * Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above)
-* If you use the native connection, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for specific steps
+* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for specific steps
 
 Configure the environment variables and check the command.
 

docs/en/12-taos-sql/05-insert.md

@@ -159,7 +159,7 @@ Automatically creating table and the table name is specified through the `tbname
 ```sql
 INSERT INTO meters(tbname, location, groupId, ts, current, phase) 
                 values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) 
-                values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
-                values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
+                ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
+                ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
 ```
 

docs/en/12-taos-sql/26-udf.md

@@ -53,7 +53,7 @@ CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [
   CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 64;
   ``` 
 
-For more information about user-defined functions, see [User-Defined Functions](/develop/udf).
+For more information about user-defined functions, see [User-Defined Functions](../../develop/udf).
 
 ## Manage UDF
 

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -262,6 +262,63 @@ The following types may be returned:
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
+
+Prepare data
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+Execute query
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+Return results
+
+```json
+{
+     "code": 0,
+     "column_meta": [
+         [
+             "ts",
+             "TIMESTAMP",
+             8
+         ],
+         [
+             "c1",
+             "VARBINARY",
+             20
+         ],
+         [
+             "c2",
+             "GEOMETRY",
+             100
+         ]
+     ],
+     "data": [
+         [
+             "2023-11-01T06:28:15.210Z",
+             "7f8290",
+             "010100000000000000000059400000000000005940"
+         ]
+     ],
+     "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
 
 #### Errors
 

docs/examples/csharp/.gitignore

@@ -1,5 +1,5 @@
 .vs
-asyncQuery/bin
+.idea
 connect/bin
 influxdbLine/bin
 optsJSON/bin
@@ -12,7 +12,6 @@ wsConnect/bin
 wsInsert/bin
 wsQuery/bin
 wsStmt/bin
-asyncQuery/obj
 connect/obj
 influxdbLine/obj
 optsJSON/obj

docs/examples/csharp/asyncQuery/Program.cs

@@ -1,111 +0,0 @@
-using System;
-using System.Collections.Generic;
-using TDengineDriver;
-using TDengineDriver.Impl;
-using System.Runtime.InteropServices;
-
-namespace TDengineExample
-{
-    public class AsyncQueryExample
-    {
-        static void Main()
-        {
-            IntPtr conn = GetConnection();
-            try
-            {
-                QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback);
-                TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero);
-                Thread.Sleep(2000);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static void QueryCallback(IntPtr param, IntPtr taosRes, int code)
-        {
-            if (code == 0 && taosRes != IntPtr.Zero)
-            {
-                FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback);
-                TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param);
-            }
-            else
-            {
-                throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}");
-            }
-        }
-
-        // Iteratively call this interface until "numOfRows" is no greater than 0.
-        static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows)
-        {
-            if (numOfRows > 0)
-            {
-                Console.WriteLine($"{numOfRows} rows async retrieved");
-                IntPtr pdata = TDengine.GetRawBlock(taosRes);
-                List<TDengineMeta> metaList = TDengine.FetchFields(taosRes);
-                List<object> dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows);
-
-                for (int i = 0; i < dataList.Count; i++)
-                {
-                    if (i != 0 && (i + 1) % metaList.Count == 0)
-                    {
-                        Console.WriteLine("{0}\t|", dataList[i]);
-                    }
-                    else
-                    {
-                        Console.Write("{0}\t|", dataList[i]);
-                    }
-                }
-                Console.WriteLine("");
-                TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param);
-            }
-            else
-            {
-                if (numOfRows == 0)
-                {
-                    Console.WriteLine("async retrieve complete.");
-                }
-                else
-                {
-                    throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}");
-                }
-                TDengine.FreeResult(taosRes);
-            }
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-    }
-}
-
-// //output:
-// // Connect to TDengine success
-// // 8 rows async retrieved
-
-// // 1538548685500   |       11.8    |       221     |       0.28    |       california.losangeles   |       2       |
-// // 1538548696600   |       13.4    |       223     |       0.29    |       california.losangeles   |       2       |
-// // 1538548685000   |       10.8    |       223     |       0.29    |       california.losangeles   |       3       |
-// // 1538548686500   |       11.5    |       221     |       0.35    |       california.losangeles   |       3       |
-// // 1538548685000   |       10.3    |       219     |       0.31    |       california.sanfrancisco         |       2       |
-// // 1538548695000   |       12.6    |       218     |       0.33    |       california.sanfrancisco         |       2       |
-// // 1538548696800   |       12.3    |       221     |       0.31    |       california.sanfrancisco         |       2       |
-// // 1538548696650   |       10.3    |       218     |       0.25    |       california.sanfrancisco         |       3       |
-// // async retrieve complete.

docs/examples/csharp/asyncQuery/asyncquery.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <StartupObject>TDengineExample.AsyncQueryExample</StartupObject>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
-  </ItemGroup>
-
-</Project>

docs/examples/csharp/connect/Program.cs

@@ -1,4 +1,5 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
@@ -7,23 +8,11 @@ namespace TDengineExample
     {
         static void Main(String[] args)
         {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception("Connect to TDengine failed");
+                Console.WriteLine("connected");
             }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            TDengine.Close(conn);
-            TDengine.Cleanup();
         }
     }
 }

docs/examples/csharp/connect/connect.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/csharp.sln

@@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.30114.105
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}"
@@ -38,10 +36,6 @@ Global
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU

docs/examples/csharp/influxdbLine/Program.cs

@@ -1,4 +1,5 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
@@ -6,60 +7,23 @@ namespace TDengineExample
     {
         static void Main()
         {
-            IntPtr conn = GetConnection();
-            PrepareDatabase(conn);
-            string[] lines = {
-                "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
-                "meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
-                "meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
-                "meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
-            };
-            IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS);
-            if (TDengine.ErrorNo(res) != 0)
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
-            }
-            else
-            {
-                int affectedRows = TDengine.AffectRows(res);
-                Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-            }
-            TDengine.FreeResult(res);
-
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("use test");
+                string[] lines = {
+                    "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
+                    "meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
+                    "meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
+                    "meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
+                };
+                client.SchemalessInsert(lines,
+                    TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
         }
-
     }
 
 }

docs/examples/csharp/influxdbLine/influxdbline.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/optsJSON/Program.cs

@@ -1,69 +1,28 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class OptsJsonExample
     {
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                PrepareDatabase(conn);
-                string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
-                " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
-                "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
-                " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
-            };
-
-                IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
-                if (TDengine.ErrorNo(res) != 0)
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("use test");
+                string[] lines =
                 {
-                    throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
-                }
-                else
-                {
-                    int affectedRows = TDengine.AffectRows(res);
-                    Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-                }
-                TDengine.FreeResult(res);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
+                    "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
+                    " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
+                    "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
+                    " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
+                };
+                client.SchemalessInsert(lines, TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
         }
     }
-}
+}

docs/examples/csharp/optsJSON/optsJSON.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/optsTelnet/Program.cs

@@ -1,72 +1,31 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class OptsTelnetExample
     {
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                PrepareDatabase(conn);
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("USE test");
                 string[] lines = {
-                "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
-                "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
-                "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3",
-                "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3",
-                "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2",
-                "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2",
-                "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
-                "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
-            };
-                IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
-                if (TDengine.ErrorNo(res) != 0)
-                {
-                    throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
-                }
-                else
-                {
-                    int affectedRows = TDengine.AffectRows(res);
-                    Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-                }
-                TDengine.FreeResult(res);
-            }
-            catch
-            {
-                TDengine.Close(conn);
-            }
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
+                    "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
+                    "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
+                    "meters.current 1648432611249 10.8 location=California.LosAngeles groupid=3",
+                    "meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3",
+                    "meters.voltage 1648432611249 219 location=California.SanFrancisco groupid=2",
+                    "meters.voltage 1648432611250 218 location=California.SanFrancisco groupid=2",
+                    "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
+                    "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
+                };
+                client.SchemalessInsert(lines,
+                    TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
         }
     }

docs/examples/csharp/optsTelnet/optstelnet.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/query/Program.cs

@@ -1,80 +1,35 @@
-using TDengineDriver;
-using TDengineDriver.Impl;
-using System.Runtime.InteropServices;
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class QueryExample
     {
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                // run query
-                IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2");
-                if (TDengine.ErrorNo(res) != 0)
+                try
                 {
-                    throw new Exception("Failed to query since: " + TDengine.Error(res));
-                }
-
-                // get filed count
-                int fieldCount = TDengine.FieldCount(res);
-                Console.WriteLine("fieldCount=" + fieldCount);
-
-                // print column names
-                List<TDengineMeta> metas = LibTaos.GetMeta(res);
-                for (int i = 0; i < metas.Count; i++)
-                {
-                    Console.Write(metas[i].name + "\t");
-                }
-                Console.WriteLine();
-
-                // print values
-                List<Object> resData = LibTaos.GetData(res);
-                for (int i = 0; i < resData.Count; i++)
-                {
-                    Console.Write($"|{resData[i].ToString()} \t");
-                    if (((i + 1) % metas.Count == 0))
+                    client.Exec("use power");
+                    string query = "SELECT * FROM meters";
+                    using (var rows = client.Query(query))
                     {
-                        Console.WriteLine("");
+                        while (rows.Read())
+                        {
+                            Console.WriteLine(
+                                $"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
+                        }
                     }
                 }
-                Console.WriteLine();
-
-                // Free result after use
-                TDengine.FreeResult(res);
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
             }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
         }
     }
-}
-
-// output:
-// Connect to TDengine success
-// fieldCount=6
-// ts      current voltage phase   location        groupid
-// 1648432611249   10.3    219     0.31    California.SanFrancisco        2
-// 1648432611749   12.6    218     0.33    California.SanFrancisco        2
+}

docs/examples/csharp/query/query.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/sqlInsert/Program.cs

@@ -1,69 +1,47 @@
-using TDengineDriver;
-
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class SQLInsertExample
     {
 
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
-                CheckRes(conn, res, "failed to create database");
-                res = TDengine.Query(conn, "USE power");
-                CheckRes(conn, res, "failed to change database");
-                res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
-                CheckRes(conn, res, "failed to create stable");
-                var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
-                            "d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
-                            "d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
-                            "d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
-                res = TDengine.Query(conn, sql);
-                CheckRes(conn, res, "failed to insert data");
-                int affectedRows = TDengine.AffectRows(res);
-                Console.WriteLine("affectedRows " + affectedRows);
-                TDengine.FreeResult(res);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void CheckRes(IntPtr conn, IntPtr res, String errorMsg)
-        {
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception($"{errorMsg} since: {TDengine.Error(res)}");
+                try
+                {
+                    client.Exec("create database power");
+                    client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    string insertQuery =
+                        "INSERT INTO " +
+                        "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
+                        "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
+                        "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
+                        "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
+                        "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
+                        "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
+                        "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
+                        "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
+                    client.Exec(insertQuery);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
             }
         }
-
     }
 }
-
-// output:
-// Connect to TDengine success
-// affectedRows 8

docs/examples/csharp/sqlInsert/sqlinsert.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/stmtInsert/Program.cs

@@ -1,109 +1,38 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class StmtInsertExample
     {
-        private static IntPtr conn;
-        private static IntPtr stmt;
-        static void Main()
+        public static void Main(string[] args)
         {
-            conn = GetConnection();
-            try
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                PrepareSTable();
-                // 1. init and prepare
-                stmt = TDengine.StmtInit(conn);
-                if (stmt == IntPtr.Zero)
+                try
                 {
-                    throw new Exception("failed to init stmt.");
+                    client.Exec($"create database power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    using (var stmt = client.StmtInit())
+                    {
+                        stmt.Prepare(
+                            "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
+                        var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
+                        stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
+                        stmt.AddBatch();
+                        stmt.Exec();
+                        var affected = stmt.Affected();
+                        Console.WriteLine($"affected rows: {affected}");
+                    }
                 }
-                int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)");
-                CheckStmtRes(res, "failed to prepare stmt");
-
-                // 2. bind table name and tags
-                TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) };
-                res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags);
-                CheckStmtRes(res, "failed to bind table name and tags");
-
-                // 3. bind values
-                TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] {
-                TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}),
-                TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}),
-                TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}),
-                TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f})
-            };
-                res = TDengine.StmtBindParamBatch(stmt, values);
-                CheckStmtRes(res, "failed to bind params");
-
-                // 4. add batch
-                res = TDengine.StmtAddBatch(stmt);
-                CheckStmtRes(res, "failed to add batch");
-
-                // 5. execute
-                res = TDengine.StmtExecute(stmt);
-                CheckStmtRes(res, "failed to execute");
-
-                // 6. free 
-                TaosMultiBind.FreeTaosBind(tags);
-                TaosMultiBind.FreeTaosBind(values);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareSTable()
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
-            CheckResPtr(res, "failed to create database");
-            res = TDengine.Query(conn, "USE power");
-            CheckResPtr(res, "failed to change database");
-            res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
-            CheckResPtr(res, "failed to create stable");
-        }
-
-        static void CheckStmtRes(int res, string errorMsg)
-        {
-            if (res != 0)
-            {
-                Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt));
-                int code = TDengine.StmtClose(stmt);
-                if (code != 0)
+                catch (Exception e)
                 {
-                    throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} ");
+                    Console.WriteLine(e);
+                    throw;
                 }
             }
         }
-
-        static void CheckResPtr(IntPtr res, string errorMsg)
-        {
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception(errorMsg + " since:" + TDengine.Error(res));
-            }
-        }
-
     }
-}
+}

docs/examples/csharp/stmtInsert/stmtinsert.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/subscribe/Program.cs

@@ -1,95 +1,72 @@
-using System;
-using TDengineTMQ;
-using TDengineDriver;
-using System.Runtime.InteropServices;
+
+using TDengine.Driver;
+using TDengine.Driver.Client;
+using TDengine.TMQ;
 
 namespace TMQExample
 {
     internal class SubscribeDemo
     {
-        static void Main(string[] args)
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            string topic = "topic_example";
-            //create topic 
-            IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters");
-
-            if (TDengine.ErrorNo(res) != 0 )
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception($"create topic failed, reason:{TDengine.Error(res)}");
-            }
-
-            var cfg = new ConsumerConfig
-            {
-                GourpId = "group_1",
-                TDConnectUser = "root",
-                TDConnectPasswd = "taosdata",
-                MsgWithTableName = "true",
-                TDConnectIp = "127.0.0.1",
-            };
-
-            // create consumer 
-            var consumer = new ConsumerBuilder(cfg)
-                .Build();
-
-            // subscribe
-            consumer.Subscribe(topic);
-
-            // consume 
-            for (int i = 0; i < 5; i++)
-            {
-                var consumeRes = consumer.Consume(300);
-                // print consumeResult
-                foreach (KeyValuePair<TopicPartition, TaosResult> kv in consumeRes.Message)
+                try
                 {
-                    Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString());
-
-                    kv.Value.Metas.ForEach(meta =>
+                    client.Exec("CREATE DATABASE power");
+                    client.Exec("USE power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    client.Exec("CREATE TOPIC topic_meters as SELECT * from power.meters");
+                    var cfg = new Dictionary<string, string>()
                     {
-                        Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size);
-                    });
-                    Console.WriteLine("");
-                    kv.Value.Datas.ForEach(data =>
+                        { "group.id", "group1" },
+                        { "auto.offset.reset", "latest" },
+                        { "td.connect.ip", "127.0.0.1" },
+                        { "td.connect.user", "root" },
+                        { "td.connect.pass", "taosdata" },
+                        { "td.connect.port", "6030" },
+                        { "client.id", "tmq_example" },
+                        { "enable.auto.commit", "true" },
+                        { "msg.with.table.name", "false" },
+                    };
+                    var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
+                    consumer.Subscribe(new List<string>() { "topic_meters" });
+                    Task.Run(InsertData);
+                    while (true)
                     {
-                        Console.WriteLine(data.ToString());
-                    });
+                        using (var cr = consumer.Consume(500))
+                        {
+                            if (cr == null) continue;
+                            foreach (var message in cr.Message)
+                            {
+                                Console.WriteLine(
+                                    $"message {{{((DateTime)message.Value["ts"]).ToString("yyyy-MM-dd HH:mm:ss.fff")}, " +
+                                    $"{message.Value["current"]}, {message.Value["voltage"]}, {message.Value["phase"]}}}");
+                            }
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    throw;
                 }
-
-                consumer.Commit(consumeRes);
-                Console.WriteLine("\n================ {0} done ", i);
-
             }
-
-            // retrieve topic list
-            List<string> topics = consumer.Subscription();
-            topics.ForEach(t => Console.WriteLine("topic name:{0}", t));
-
-            // unsubscribe
-            consumer.Unsubscribe();
-
-            // close consumer after use.Otherwise will lead memory leak.
-            consumer.Close();
-            TDengine.Close(conn);
-
         }
 
-        static IntPtr GetConnection()
+        static void InsertData()
         {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception("Connect to TDengine failed");
+                while (true)
+                {
+                    client.Exec("INSERT into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(now,11.5,219,0.30)");
+                    Task.Delay(1000).Wait();
+                }
             }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
         }
     }
 

docs/examples/csharp/subscribe/subscribe.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/wsConnect/Program.cs

@@ -1,28 +1,18 @@
 using System;
-using TDengineWS.Impl;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSConnExample
     {
-        static int Main(string[] args)
+        static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-  
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
+                Console.WriteLine("connected");
             }
-            else
-            {
-                Console.WriteLine("Establish connect success.");
-                // close connection.
-                LibTaosWS.WSClose(wsConn);
-            }
-
-            return 0;
         }
     }
 }

docs/examples/csharp/wsConnect/wsConnect.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsInsert/Program.cs

@@ -1,61 +1,46 @@
 using System;
-using TDengineWS.Impl;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSInsertExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-
-            // Assert if connection is validate
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
-            }
-            else
-            {
-                Console.WriteLine("Establish connect success.");
-            }
-
-            string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);";
-            string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" +
-                            "test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" +
-                            "test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
-                            "test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
-
-            IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable);
-            ValidInsert("create table", wsRes);
-            LibTaosWS.WSFreeResult(wsRes);
-
-            wsRes = LibTaosWS.WSQuery(wsConn, insert);
-            ValidInsert("insert data", wsRes);
-            LibTaosWS.WSFreeResult(wsRes);
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
-        }
-
-        static void ValidInsert(string desc, IntPtr wsRes)
-        {
-            int code = LibTaosWS.WSErrorNo(wsRes);
-            if (code != 0)
-            {
-                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
-            }
-            else
-            {
-                Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes));
+                try
+                {
+                    client.Exec("create database power");
+                    client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    string insertQuery =
+                        "INSERT INTO " +
+                        "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
+                        "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
+                        "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
+                        "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
+                        "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
+                        "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
+                        "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
+                        "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
+                    client.Exec(insertQuery);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
             }
         }
     }
-
 }
-// Establish connect success.
-// create table success affect 0 rows, cost 3717542 nanoseconds
-// insert data success affect 8 rows, cost 2613637 nanoseconds

docs/examples/csharp/wsInsert/wsInsert.csproj

@@ -6,7 +6,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsQuery/Program.cs

@@ -1,79 +1,36 @@
 using System;
-using TDengineWS.Impl;
-using System.Collections.Generic;
-using TDengineDriver;
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSQueryExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
-            }
-            else
-            {
-                Console.WriteLine("Establish connect success.");
-            }
-
-            string select = "select * from test.meters";
-
-            // optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
-            IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
-            // Assert if query execute success.
-            int code = LibTaosWS.WSErrorNo(wsRes);
-            if (code != 0)
-            {
-                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
-                LibTaosWS.WSFreeResult(wsRes);
-                return -1;
-            }
-
-            // get meta data
-            List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
-            // get retrieved data
-            List<object> dataSet = LibTaosWS.WSGetData(wsRes);
-
-            // do something with result.
-            foreach (var meta in metas)
-            {
-                Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
-            }
-            Console.WriteLine("");
-
-            for (int i = 0; i < dataSet.Count;)
-            {
-                for (int j = 0; j < metas.Count; j++)
+                try
                 {
-                    Console.Write("{0}\t|\t", dataSet[i]);
-                    i++;
+                    client.Exec("use power");
+                    string query = "SELECT * FROM meters";
+                    using (var rows = client.Query(query))
+                    {
+                        while (rows.Read())
+                        {
+                            Console.WriteLine(
+                                $"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    throw;
                 }
-                Console.WriteLine("");
             }
-
-            // Free result after use.
-            LibTaosWS.WSFreeResult(wsRes);
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
         }
     }
-}
-
-// Establish connect success.
-// ts TIMESTAMP(8)         |       current FLOAT(4)        |       voltage INT(4)  |       phase FLOAT(4)  |       location BINARY(64)     |       groupid INT(4)  |
-// 1538548685000   |       10.8    |       223     |       0.29    |       California.LosAngeles   |       3       |
-// 1538548686500   |       11.5    |       221     |       0.35    |       California.LosAngeles   |       3       |
-// 1538548685500   |       11.8    |       221     |       0.28    |       California.LosAngeles   |       2       |
-// 1538548696600   |       13.4    |       223     |       0.29    |       California.LosAngeles   |       2       |
-// 1538548685000   |       10.3    |       219     |       0.31    |       California.SanFrancisco |       2       |
-// 1538548695000   |       12.6    |       218     |       0.33    |       California.SanFrancisco |       2       |
-// 1538548696800   |       12.3    |       221     |       0.31    |       California.SanFrancisco |       2       |
-// 1538548696650   |       10.3    |       218     |       0.25    |       California.SanFrancisco |       3       |
+}

docs/examples/csharp/wsQuery/wsQuery.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsStmt/Program.cs

@@ -1,98 +1,41 @@
 using System;
-using TDengineWS.Impl;
-using TDengineDriver;
-using System.Runtime.InteropServices;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSStmtExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            const string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            const string table = "meters";
-            const string database = "test";
-            const string childTable = "d1005";
-            string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
-            const int numOfTags = 2;
-            const int numOfColumns = 4;
-
-            // Establish connection
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-            if (wsConn == IntPtr.Zero)
+            var builder =
+                new ConnectionStringBuilder(
+                    "protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine($"get WS connection failed");
-                return -1;
-            }
-            else
-            {
-                Console.WriteLine("Establish connect success...");
-            }
-
-            // init stmt
-            IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn);
-            if (wsStmt != IntPtr.Zero)
-            {
-                int code = LibTaosWS.WSStmtPrepare(wsStmt, insert);
-                ValidStmtStep(code, wsStmt, "WSStmtPrepare");
-
-                TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) };
-                code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags);
-                ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags");
-
-                TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4];
-                data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 });
-                data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F });
-                data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
-                data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
-                code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
-                ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
-
-                code = LibTaosWS.WSStmtAddBatch(wsStmt);
-                ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
-
-                IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
-                code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
-                ValidStmtStep(code, wsStmt, "WSStmtExecute");
-                Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
-                Marshal.FreeHGlobal(stmtAffectRowPtr);
-
-                LibTaosWS.WSStmtClose(wsStmt);
-
-                // Free unmanaged memory
-                WSMultiBind.WSFreeTaosBind(wsTags);
-                WSMultiBind.WSFreeTaosBind(data);
-
-                //check result with SQL "SELECT * FROM test.d1005;"
-            }
-            else
-            {
-                Console.WriteLine("Init STMT failed...");
-            }
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
-        }
-
-        static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
-        {
-            if (code != 0)
-            {
-                Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
-            }
-            else
-            {
-                Console.WriteLine("{0} success...", desc);
+                try
+                {
+                    client.Exec($"create database power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    using (var stmt = client.StmtInit())
+                    {
+                        stmt.Prepare(
+                            "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
+                        var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
+                        stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
+                        stmt.AddBatch();
+                        stmt.Exec();
+                        var affected = stmt.Affected();
+                        Console.WriteLine($"affected rows: {affected}");
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e);
+                    throw;
+                }
             }
         }
     }
-}
-
-// WSStmtPrepare success...
-// WSStmtSetTbnameTags success...
-// WSStmtBindParamBatch success...
-// WSStmtAddBatch success...
-// WSStmtExecute success...
-// WS STMT insert 5 rows...
+}

docs/examples/csharp/wsStmt/wsStmt.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/zh/07-develop/01-connect/index.md

@@ -176,7 +176,7 @@ npm install @tdengine/rest
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.0" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.0" />
   </ItemGroup>
 
 </Project>

docs/zh/07-develop/04-query-data/_cs_async.mdx

@@ -1,3 +0,0 @@
-```csharp
-{{#include docs/examples/csharp/asyncQuery/Program.cs}}
-```

docs/zh/07-develop/04-query-data/index.mdx

@@ -16,7 +16,6 @@ import CQuery from "./_c.mdx";
 import PhpQuery from "./_php.mdx";
 import PyAsync from "./_py_async.mdx";
 import NodeAsync from "./_js_async.mdx";
-import CsAsync from "./_cs_async.mdx";
 import CAsync from "./_c_async.mdx";
 
 ## 主要查询功能
@@ -175,9 +174,6 @@ Query OK, 6 rows in database (0.005515s)
   <TabItem label="Python" value="python">
     <PyAsync />
   </TabItem>
-  <TabItem label="C#" value="csharp">
-    <CsAsync />
-  </TabItem>
   <TabItem label="C" value="c">
     <CAsync />
   </TabItem>

docs/zh/07-develop/07-tmq.mdx

@@ -249,23 +249,23 @@ function close()
 <TabItem value="C#" label="C#">
 
 ```csharp
+class ConsumerBuilder<TValue>
+
 ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
 
-virtual IConsumer Build()
-
-Consumer(ConsumerBuilder builder)
+public IConsumer<TValue> Build()
 
 void Subscribe(IEnumerable<string> topics)
 
 void Subscribe(string topic) 
 
-ConsumeResult Consume(int millisecondsTimeout)
+ConsumeResult<TValue> Consume(int millisecondsTimeout)
 
 List<string> Subscription()
 
 void Unsubscribe()
  
-void Commit(ConsumeResult consumerResult)
+List<TopicPartitionOffset> Commit()
 
 void Close()
 ```
@@ -502,25 +502,19 @@ let consumer = taos.consumer({
 <TabItem value="C#" label="C#">
 
 ```csharp
-using TDengineTMQ;
-
-// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、
-// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数
-var cfg = new ConsumerConfig
- {
-    EnableAutoCommit = "true"
-    AutoCommitIntervalMs = "1000"
-    GourpId = "TDengine-TMQ-C#",
-    TDConnectUser = "root",
-    TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "latest"
-    MsgWithTableName = "true",
-    TDConnectIp = "127.0.0.1",
-    TDConnectPort = "6030"
- };
-
-var consumer = new ConsumerBuilder(cfg).Build();
-
+var cfg = new Dictionary<string, string>()
+{
+    { "group.id", "group1" },
+    { "auto.offset.reset", "latest" },
+    { "td.connect.ip", "127.0.0.1" },
+    { "td.connect.user", "root" },
+    { "td.connect.pass", "taosdata" },
+    { "td.connect.port", "6030" },
+    { "client.id", "tmq_example" },
+    { "enable.auto.commit", "true" },
+    { "msg.with.table.name", "false" },
+};
+var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
 ```
 
 </TabItem>
@@ -749,10 +743,12 @@ while(true){
 // 消费数据
 while (true)
 {
-    var consumerRes = consumer.Consume(100);
-    // process ConsumeResult
-    ProcessMsg(consumerRes);
-    consumer.Commit(consumerRes);
+    using (var result = consumer.Consume(500))
+    {
+        if (result == null) continue;
+        ProcessMsg(result);
+        consumer.Commit();
+    }
 }
 ```
 

docs/zh/08-connector/02-rest-api.mdx

@@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
+
+准备数据
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+执行查询
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+返回结果
+
+```json
+{
+    "code": 0,
+    "column_meta": [
+        [
+            "ts",
+            "TIMESTAMP",
+            8
+        ],
+        [
+            "c1",
+            "VARBINARY",
+            20
+        ],
+        [
+            "c2",
+            "GEOMETRY",
+            100
+        ]
+    ],
+    "data": [
+        [
+            "2023-11-01T06:28:15.210Z",
+            "7f8290",
+            "010100000000000000000059400000000000005940"
+        ]
+    ],
+    "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)  格式
 
 #### 错误
 

docs/zh/12-taos-sql/05-insert.md

@@ -159,6 +159,6 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
 ```sql
 INSERT INTO meters(tbname, location, groupId, ts, current, phase) 
                 values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) 
-                values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
-                values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
+                ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
+                ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
 ```

include/common/cos.h

@@ -34,7 +34,7 @@ extern int32_t tsS3UploadDelaySec;
 int32_t s3Init();
 void    s3CleanUp();
 int32_t s3PutObjectFromFile(const char *file, const char *object);
-int32_t s3PutObjectFromFile2(const char *file, const char *object);
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
 void    s3DeleteObjectsByPrefix(const char *prefix);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);
@@ -43,6 +43,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
 int32_t s3GetObjectsByPrefix(const char *prefix, const char *path);
 void    s3EvictCache(const char *path, long object_size);
 long    s3Size(const char *object_name);
+int32_t s3GetObjectToFile(const char *object_name, char *fileName);
 
 #ifdef __cplusplus
 }

include/common/cos_cp.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_COS_CP_H_
+#define _TD_COMMON_COS_CP_H_
+
+#include "os.h"
+#include "tdef.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+  COS_CP_TYPE_UPLOAD,   // upload
+  COS_CP_TYPE_DOWNLOAD  // download
+} ECpType;
+
+typedef struct {
+  int32_t  index;      // the index of part, start from 0
+  int64_t  offset;     // the offset point of part
+  int64_t  size;       // the size of part
+  int      completed;  // COS_TRUE completed, COS_FALSE uncompleted
+  char     etag[128];  // the etag of part, for upload
+  uint64_t crc64;
+} SCheckpointPart;
+
+typedef struct {
+  ECpType   cp_type;  // 0 upload, 1 download
+  char      md5[64];  // the md5 of checkout content
+  TdFilePtr thefile;  // the handle of checkpoint file
+
+  char    file_path[TSDB_FILENAME_LEN];  // local file path
+  int64_t file_size;                     // local file size, for upload
+  int32_t file_last_modified;            // local file last modified time, for upload
+  char    file_md5[64];                  // md5 of the local file content, for upload, reserved
+
+  char    object_name[128];          // object name
+  int64_t object_size;               // object size, for download
+  char    object_last_modified[64];  // object last modified time, for download
+  char    object_etag[128];          // object etag, for download
+
+  char upload_id[128];  // upload id
+
+  int              part_num;   // the total number of parts
+  int64_t          part_size;  // the part size, byte
+  SCheckpointPart* parts;      // the parts of local or object, from 0
+} SCheckpoint;
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
+void    cos_cp_close(TdFilePtr fd);
+void    cos_cp_remove(char const* filepath);
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
+int32_t cos_cp_dump(SCheckpoint* checkpoint);
+void    cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
+void    cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
+void    cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                            char const* upload_id, int64_t part_size);
+bool    cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
+
+void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
+                           char const* object_lmtime, char const* object_etag, int64_t part_size);
+bool cos_cp_is_valid_download(SCheckpoint* checkpoint, char const* object_name, int64_t object_size,
+                              char const* object_lmtime, char const* object_etag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_COS_CP_H_*/

include/common/systable.h

@@ -50,6 +50,8 @@ extern "C" {
 #define TSDB_INS_TABLE_STREAM_TASKS      "ins_stream_tasks"
 #define TSDB_INS_TABLE_USER_PRIVILEGES   "ins_user_privileges"
 #define TSDB_INS_TABLE_VIEWS             "ins_views"
+#define TSDB_INS_TABLE_COMPACTS          "ins_compacts"
+#define TSDB_INS_TABLE_COMPACT_DETAILS   "ins_compact_details"
 
 #define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS        "perf_smas"

include/common/tcommon.h

@@ -253,13 +253,12 @@ typedef struct SQueryTableDataCond {
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
+  bool         notLoadData;    // response the actual data, not only the rows in the attribute of info.row of ssdatablock
 } SQueryTableDataCond;
 
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
-int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
-void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
 void    colDataDestroy(SColumnInfoData* pColData);
 
 //======================================================================================================================
@@ -294,7 +293,7 @@ typedef struct STableBlockDistInfo {
   int32_t  defMaxRows;
   int32_t  firstSeekTimeUs;
   uint32_t numOfInmemRows;
-  uint32_t numOfSmallBlocks;
+  uint32_t numOfSttRows;
   uint32_t numOfVgroups;
   int32_t  blockRowsHisto[20];
 } STableBlockDistInfo;

include/common/tglobal.h

@@ -75,12 +75,15 @@ extern int32_t tsElectInterval;
 extern int32_t tsHeartbeatInterval;
 extern int32_t tsHeartbeatTimeout;
 
+// vnode
+extern int64_t tsVndCommitMaxIntervalMs;
+
 // snode
 extern int32_t tsRsyncPort;
-extern char tsCheckpointBackupDir[];
+extern char    tsCheckpointBackupDir[];
 
 // vnode checkpoint
-extern char tsSnodeAddress[];       //127.0.0.1:873
+extern char tsSnodeAddress[];  // 127.0.0.1:873
 
 // mnode
 extern int64_t tsMndSdbWriteDelta;
@@ -114,9 +117,9 @@ extern int32_t  tsTelemInterval;
 extern char     tsTelemServer[];
 extern uint16_t tsTelemPort;
 extern bool     tsEnableCrashReport;
-extern char    *tsTelemUri;
-extern char    *tsClientCrashReportUri;
-extern char    *tsSvrCrashReportUri;
+extern char *   tsTelemUri;
+extern char *   tsClientCrashReportUri;
+extern char *   tsSvrCrashReportUri;
 
 // query buffer management
 extern int32_t tsQueryBufferSize;  // maximum allowed usage buffer size in MB for each data node during query processing
@@ -193,6 +196,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
 
 // internal
 extern int32_t tsTransPullupInterval;
+extern int32_t tsCompactPullupInterval;
 extern int32_t tsMqRebalanceInterval;
 extern int32_t tsStreamCheckpointInterval;
 extern float   tsSinkDataRate;
@@ -211,6 +215,7 @@ extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
 extern int32_t tsResolveFQDNRetryTime;
 
+extern bool tsExperimental;
 // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,

include/common/tmsg.h

@@ -65,9 +65,9 @@ typedef uint16_t tmsg_t;
 #define TMSG_INDEX(TYPE)    (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
 
 static inline bool tmsgIsValid(tmsg_t type) {
+  // static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]);
   int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG);
-
-  int segIdx = TMSG_SEG_CODE(type);
+  int    segIdx = TMSG_SEG_CODE(type);
   if (segIdx >= 0 && segIdx < maxSegIdx) {
     return type < tMsgRangeDict[segIdx];
   }
@@ -145,6 +145,8 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_STREAM_TASKS,
   TSDB_MGMT_TABLE_PRIVILEGES,
   TSDB_MGMT_TABLE_VIEWS,
+  TSDB_MGMT_TABLE_COMPACT,
+  TSDB_MGMT_TABLE_COMPACT_DETAIL,
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
@@ -296,7 +298,37 @@ typedef enum ENodeType {
   QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_GRANT_STMT,
   QUERY_NODE_REVOKE_STMT,
-  QUERY_NODE_SHOW_DNODES_STMT,
+  // placeholder for [152, 180]
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
+  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_KILL_COMPACT_STMT,
+  QUERY_NODE_DELETE_STMT,
+  QUERY_NODE_INSERT_STMT,
+  QUERY_NODE_QUERY,
+  QUERY_NODE_SHOW_DB_ALIVE_STMT,
+  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_RESTORE_DNODE_STMT,
+  QUERY_NODE_RESTORE_QNODE_STMT,
+  QUERY_NODE_RESTORE_MNODE_STMT,
+  QUERY_NODE_RESTORE_VNODE_STMT,
+  QUERY_NODE_PAUSE_STREAM_STMT,
+  QUERY_NODE_RESUME_STREAM_STMT,
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
+
+  // show statement nodes
+  // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
+  QUERY_NODE_SHOW_DNODES_STMT = 400,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
@@ -325,31 +357,8 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_VIEWS_STMT,
-  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
-  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
-  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
-  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
-  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
-  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
-  QUERY_NODE_SHOW_SCORES_STMT,
-  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
-  QUERY_NODE_KILL_CONNECTION_STMT,
-  QUERY_NODE_KILL_QUERY_STMT,
-  QUERY_NODE_KILL_TRANSACTION_STMT,
-  QUERY_NODE_DELETE_STMT,
-  QUERY_NODE_INSERT_STMT,
-  QUERY_NODE_QUERY,
-  QUERY_NODE_SHOW_DB_ALIVE_STMT,
-  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
-  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
-  QUERY_NODE_RESTORE_DNODE_STMT,
-  QUERY_NODE_RESTORE_QNODE_STMT,
-  QUERY_NODE_RESTORE_MNODE_STMT,
-  QUERY_NODE_RESTORE_VNODE_STMT,
-  QUERY_NODE_PAUSE_STREAM_STMT,
-  QUERY_NODE_RESUME_STREAM_STMT,
-  QUERY_NODE_CREATE_VIEW_STMT,
-  QUERY_NODE_DROP_VIEW_STMT,
+  QUERY_NODE_SHOW_COMPACTS_STMT,
+  QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@@ -1381,6 +1390,24 @@ int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
 int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
 void    tFreeSCompactDbReq(SCompactDbReq* pReq);
 
+typedef struct {
+  int32_t compactId;
+  int8_t bAccepted;
+} SCompactDbRsp;
+
+int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
+int32_t tDeserializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
+
+typedef struct {
+  int32_t compactId;
+  int32_t sqlLen;
+  char*   sql;
+} SKillCompactReq;
+
+int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
+int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
+void    tFreeSKillCompactReq(SKillCompactReq *pReq);
+
 typedef struct {
   char    name[TSDB_FUNC_NAME_LEN];
   int8_t  igExists;
@@ -1659,6 +1686,26 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
 int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
 int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq);
 
+typedef struct {
+  int32_t compactId;
+  int32_t vgId;
+  int32_t dnodeId;
+} SQueryCompactProgressReq;
+
+int32_t tSerializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
+int32_t tDeserializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
+
+typedef struct {
+  int32_t compactId;
+  int32_t vgId;
+  int32_t dnodeId;
+  int32_t numberFileset;
+  int32_t finished;
+} SQueryCompactProgressRsp;
+
+int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
+int32_t tDeserializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
+
 typedef struct {
   int32_t vgId;
   int32_t dnodeId;
@@ -1686,11 +1733,21 @@ typedef struct {
   char        db[TSDB_DB_FNAME_LEN];
   int64_t     compactStartTime;
   STimeWindow tw;
+  int32_t     compactId;
 } SCompactVnodeReq;
 
 int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
 int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
 
+typedef struct {
+  int32_t     compactId;
+  int32_t     vgId;
+  int32_t     dnodeId;
+} SVKillCompactReq;
+
+int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
+int32_t tDeserializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
+
 typedef struct {
   int32_t vgVersion;
   int32_t buffer;
@@ -1886,8 +1943,9 @@ typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
   char    user[TSDB_USER_LEN];
-  char    filterTb[TSDB_TABLE_NAME_LEN];
+  char    filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
   int64_t showId;
+  int64_t compactId; // for compact
 } SRetrieveTableReq;
 
 typedef struct SSysTableSchema {

include/common/tmsgdef.h

@@ -38,7 +38,7 @@
   #undef TD_NEW_MSG_SEG
   #undef TD_DEF_MSG_TYPE
   #undef TD_CLOSE_MSG_SEG
-  #define TD_NEW_MSG_SEG(TYPE) 
+  #define TD_NEW_MSG_SEG(TYPE)
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
   #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
   int32_t tMsgRangeDict[] = {
@@ -61,7 +61,7 @@
   #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_SEG(TYPE)
+  #define TD_CLOSE_MSG_SEG(type)
 
   int32_t tMsgDict[] = {
 
@@ -78,17 +78,16 @@
   enum {
 #else
 
-#undef TD_NEW_MSG_SEG
-#undef TD_DEF_MSG_TYPE
-#undef TD_CLOSE_MSG_SEG
-#define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
-#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
-#define TD_CLOSE_MSG_SEG(TYPE) TYPE,
+  #undef TD_NEW_MSG_SEG
+  #undef TD_DEF_MSG_TYPE
+  #undef TD_CLOSE_MSG_SEG
+  #define TD_NEW_MSG_SEG(TYPE) TYPE = ((TYPE##_SEG_CODE) << 8),
+  #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
+  #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
 
-
-enum { // WARN: new msg should be appended to segment tail
+  enum { // WARN: new msg should be appended to segment tail
 #endif
-  TD_NEW_MSG_SEG(TDMT_DND_MSG)
+  TD_NEW_MSG_SEG(TDMT_DND_MSG)    // 0<<8
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL)
@@ -109,9 +108,8 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP, "dnode-check-vnode-learner-catchup", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG)
-  
 
-  TD_NEW_MSG_SEG(TDMT_MND_MSG)
+  TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "create-acct", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "alter-acct", NULL, NULL)
@@ -210,16 +208,19 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
-  TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "create-table", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "alter-table", NULL, NULL)
@@ -266,10 +267,12 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
-  TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG) 
+  TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_SCH_MSG)
+  TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MERGE_QUERY, "merge-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_CONTINUE, "query-continue", NULL, NULL)
@@ -285,13 +288,13 @@ enum { // WARN: new msg should be appended to segment tail
   TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG)
 
 
-  TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
-  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
+  TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
+  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)  //1025 1026
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)  //1035 1036
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT_READY, "stream-checkpoint-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL)
@@ -302,12 +305,11 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_MON_MSG)
+  TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG)
 
-
-  TD_NEW_MSG_SEG(TDMT_SYNC_MSG)
+  TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used
@@ -339,7 +341,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY_FINISH, "vnode-stream-scan-history-finish", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL)
@@ -349,7 +351,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset)
@@ -365,7 +367,8 @@ enum { // WARN: new msg should be appended to segment tail
 
   TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
   TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG)
-#if defined(TD_MSG_NUMBER_)
-  TDMT_MAX
-#endif
+
+  #if defined(TD_MSG_NUMBER_)
+    TDMT_MAX
+  #endif
 };

include/common/ttokendef.h

@@ -189,184 +189,180 @@
 #define TK_ALIVE                          170
 #define TK_VIEWS                          171
 #define TK_VIEW                           172
-#define TK_NORMAL                         173
-#define TK_CHILD                          174
-#define TK_LIKE                           175
-#define TK_TBNAME                         176
-#define TK_QTAGS                          177
-#define TK_AS                             178
-#define TK_SYSTEM                         179
-#define TK_INDEX                          180
-#define TK_FUNCTION                       181
-#define TK_INTERVAL                       182
-#define TK_COUNT                          183
-#define TK_LAST_ROW                       184
-#define TK_META                           185
-#define TK_ONLY                           186
-#define TK_TOPIC                          187
-#define TK_CONSUMER                       188
-#define TK_GROUP                          189
-#define TK_DESC                           190
-#define TK_DESCRIBE                       191
-#define TK_RESET                          192
-#define TK_QUERY                          193
-#define TK_CACHE                          194
-#define TK_EXPLAIN                        195
-#define TK_ANALYZE                        196
-#define TK_VERBOSE                        197
-#define TK_NK_BOOL                        198
-#define TK_RATIO                          199
-#define TK_NK_FLOAT                       200
-#define TK_OUTPUTTYPE                     201
-#define TK_AGGREGATE                      202
-#define TK_BUFSIZE                        203
-#define TK_LANGUAGE                       204
-#define TK_REPLACE                        205
-#define TK_STREAM                         206
-#define TK_INTO                           207
-#define TK_PAUSE                          208
-#define TK_RESUME                         209
-#define TK_TRIGGER                        210
-#define TK_AT_ONCE                        211
-#define TK_WINDOW_CLOSE                   212
-#define TK_IGNORE                         213
-#define TK_EXPIRED                        214
-#define TK_FILL_HISTORY                   215
-#define TK_UPDATE                         216
-#define TK_SUBTABLE                       217
-#define TK_UNTREATED                      218
-#define TK_KILL                           219
-#define TK_CONNECTION                     220
-#define TK_TRANSACTION                    221
-#define TK_BALANCE                        222
-#define TK_VGROUP                         223
-#define TK_LEADER                         224
-#define TK_MERGE                          225
-#define TK_REDISTRIBUTE                   226
-#define TK_SPLIT                          227
-#define TK_DELETE                         228
-#define TK_INSERT                         229
-#define TK_NULL                           230
-#define TK_NK_QUESTION                    231
-#define TK_NK_ALIAS                       232
-#define TK_NK_ARROW                       233
-#define TK_ROWTS                          234
-#define TK_QSTART                         235
-#define TK_QEND                           236
-#define TK_QDURATION                      237
-#define TK_WSTART                         238
-#define TK_WEND                           239
-#define TK_WDURATION                      240
-#define TK_IROWTS                         241
-#define TK_ISFILLED                       242
-#define TK_CAST                           243
-#define TK_NOW                            244
-#define TK_TODAY                          245
-#define TK_TIMEZONE                       246
-#define TK_CLIENT_VERSION                 247
-#define TK_SERVER_VERSION                 248
-#define TK_SERVER_STATUS                  249
-#define TK_CURRENT_USER                   250
-#define TK_CASE                           251
-#define TK_WHEN                           252
-#define TK_THEN                           253
-#define TK_ELSE                           254
-#define TK_BETWEEN                        255
-#define TK_IS                             256
-#define TK_NK_LT                          257
-#define TK_NK_GT                          258
-#define TK_NK_LE                          259
-#define TK_NK_GE                          260
-#define TK_NK_NE                          261
-#define TK_MATCH                          262
-#define TK_NMATCH                         263
-#define TK_CONTAINS                       264
-#define TK_IN                             265
-#define TK_JOIN                           266
-#define TK_INNER                          267
-#define TK_SELECT                         268
-#define TK_NK_HINT                        269
-#define TK_DISTINCT                       270
-#define TK_WHERE                          271
-#define TK_PARTITION                      272
-#define TK_BY                             273
-#define TK_SESSION                        274
-#define TK_STATE_WINDOW                   275
-#define TK_EVENT_WINDOW                   276
-#define TK_SLIDING                        277
-#define TK_FILL                           278
-#define TK_VALUE                          279
-#define TK_VALUE_F                        280
-#define TK_NONE                           281
-#define TK_PREV                           282
-#define TK_NULL_F                         283
-#define TK_LINEAR                         284
-#define TK_NEXT                           285
-#define TK_HAVING                         286
-#define TK_RANGE                          287
-#define TK_EVERY                          288
-#define TK_ORDER                          289
-#define TK_SLIMIT                         290
-#define TK_SOFFSET                        291
-#define TK_LIMIT                          292
-#define TK_OFFSET                         293
-#define TK_ASC                            294
-#define TK_NULLS                          295
-#define TK_ABORT                          296
-#define TK_AFTER                          297
-#define TK_ATTACH                         298
-#define TK_BEFORE                         299
-#define TK_BEGIN                          300
-#define TK_BITAND                         301
-#define TK_BITNOT                         302
-#define TK_BITOR                          303
-#define TK_BLOCKS                         304
-#define TK_CHANGE                         305
-#define TK_COMMA                          306
-#define TK_CONCAT                         307
-#define TK_CONFLICT                       308
-#define TK_COPY                           309
-#define TK_DEFERRED                       310
-#define TK_DELIMITERS                     311
-#define TK_DETACH                         312
-#define TK_DIVIDE                         313
-#define TK_DOT                            314
-#define TK_EACH                           315
-#define TK_FAIL                           316
-#define TK_FILE                           317
-#define TK_FOR                            318
-#define TK_GLOB                           319
-#define TK_ID                             320
-#define TK_IMMEDIATE                      321
-#define TK_IMPORT                         322
-#define TK_INITIALLY                      323
-#define TK_INSTEAD                        324
-#define TK_ISNULL                         325
-#define TK_KEY                            326
-#define TK_MODULES                        327
-#define TK_NK_BITNOT                      328
-#define TK_NK_SEMI                        329
-#define TK_NOTNULL                        330
-#define TK_OF                             331
-#define TK_PLUS                           332
-#define TK_PRIVILEGE                      333
-#define TK_RAISE                          334
-#define TK_RESTRICT                       335
-#define TK_ROW                            336
-#define TK_SEMI                           337
-#define TK_STAR                           338
-#define TK_STATEMENT                      339
-#define TK_STRICT                         340
-#define TK_STRING                         341
-#define TK_TIMES                          342
-#define TK_VALUES                         343
-#define TK_VARIABLE                       344
-#define TK_WAL                            345
-
-
-
-
-
+#define TK_COMPACTS                       173
+#define TK_NORMAL                         174
+#define TK_CHILD                          175
+#define TK_LIKE                           176
+#define TK_TBNAME                         177
+#define TK_QTAGS                          178
+#define TK_AS                             179
+#define TK_SYSTEM                         180
+#define TK_INDEX                          181
+#define TK_FUNCTION                       182
+#define TK_INTERVAL                       183
+#define TK_COUNT                          184
+#define TK_LAST_ROW                       185
+#define TK_META                           186
+#define TK_ONLY                           187
+#define TK_TOPIC                          188
+#define TK_CONSUMER                       189
+#define TK_GROUP                          190
+#define TK_DESC                           191
+#define TK_DESCRIBE                       192
+#define TK_RESET                          193
+#define TK_QUERY                          194
+#define TK_CACHE                          195
+#define TK_EXPLAIN                        196
+#define TK_ANALYZE                        197
+#define TK_VERBOSE                        198
+#define TK_NK_BOOL                        199
+#define TK_RATIO                          200
+#define TK_NK_FLOAT                       201
+#define TK_OUTPUTTYPE                     202
+#define TK_AGGREGATE                      203
+#define TK_BUFSIZE                        204
+#define TK_LANGUAGE                       205
+#define TK_REPLACE                        206
+#define TK_STREAM                         207
+#define TK_INTO                           208
+#define TK_PAUSE                          209
+#define TK_RESUME                         210
+#define TK_TRIGGER                        211
+#define TK_AT_ONCE                        212
+#define TK_WINDOW_CLOSE                   213
+#define TK_IGNORE                         214
+#define TK_EXPIRED                        215
+#define TK_FILL_HISTORY                   216
+#define TK_UPDATE                         217
+#define TK_SUBTABLE                       218
+#define TK_UNTREATED                      219
+#define TK_KILL                           220
+#define TK_CONNECTION                     221
+#define TK_TRANSACTION                    222
+#define TK_BALANCE                        223
+#define TK_VGROUP                         224
+#define TK_LEADER                         225
+#define TK_MERGE                          226
+#define TK_REDISTRIBUTE                   227
+#define TK_SPLIT                          228
+#define TK_DELETE                         229
+#define TK_INSERT                         230
+#define TK_NULL                           231
+#define TK_NK_QUESTION                    232
+#define TK_NK_ALIAS                       233
+#define TK_NK_ARROW                       234
+#define TK_ROWTS                          235
+#define TK_QSTART                         236
+#define TK_QEND                           237
+#define TK_QDURATION                      238
+#define TK_WSTART                         239
+#define TK_WEND                           240
+#define TK_WDURATION                      241
+#define TK_IROWTS                         242
+#define TK_ISFILLED                       243
+#define TK_CAST                           244
+#define TK_NOW                            245
+#define TK_TODAY                          246
+#define TK_TIMEZONE                       247
+#define TK_CLIENT_VERSION                 248
+#define TK_SERVER_VERSION                 249
+#define TK_SERVER_STATUS                  250
+#define TK_CURRENT_USER                   251
+#define TK_CASE                           252
+#define TK_WHEN                           253
+#define TK_THEN                           254
+#define TK_ELSE                           255
+#define TK_BETWEEN                        256
+#define TK_IS                             257
+#define TK_NK_LT                          258
+#define TK_NK_GT                          259
+#define TK_NK_LE                          260
+#define TK_NK_GE                          261
+#define TK_NK_NE                          262
+#define TK_MATCH                          263
+#define TK_NMATCH                         264
+#define TK_CONTAINS                       265
+#define TK_IN                             266
+#define TK_JOIN                           267
+#define TK_INNER                          268
+#define TK_SELECT                         269
+#define TK_NK_HINT                        270
+#define TK_DISTINCT                       271
+#define TK_WHERE                          272
+#define TK_PARTITION                      273
+#define TK_BY                             274
+#define TK_SESSION                        275
+#define TK_STATE_WINDOW                   276
+#define TK_EVENT_WINDOW                   277
+#define TK_SLIDING                        278
+#define TK_FILL                           279
+#define TK_VALUE                          280
+#define TK_VALUE_F                        281
+#define TK_NONE                           282
+#define TK_PREV                           283
+#define TK_NULL_F                         284
+#define TK_LINEAR                         285
+#define TK_NEXT                           286
+#define TK_HAVING                         287
+#define TK_RANGE                          288
+#define TK_EVERY                          289
+#define TK_ORDER                          290
+#define TK_SLIMIT                         291
+#define TK_SOFFSET                        292
+#define TK_LIMIT                          293
+#define TK_OFFSET                         294
+#define TK_ASC                            295
+#define TK_NULLS                          296
+#define TK_ABORT                          297
+#define TK_AFTER                          298
+#define TK_ATTACH                         299
+#define TK_BEFORE                         300
+#define TK_BEGIN                          301
+#define TK_BITAND                         302
+#define TK_BITNOT                         303
+#define TK_BITOR                          304
+#define TK_BLOCKS                         305
+#define TK_CHANGE                         306
+#define TK_COMMA                          307
+#define TK_CONCAT                         308
+#define TK_CONFLICT                       309
+#define TK_COPY                           310
+#define TK_DEFERRED                       311
+#define TK_DELIMITERS                     312
+#define TK_DETACH                         313
+#define TK_DIVIDE                         314
+#define TK_DOT                            315
+#define TK_EACH                           316
+#define TK_FAIL                           317
+#define TK_FILE                           318
+#define TK_FOR                            319
+#define TK_GLOB                           320
+#define TK_ID                             321
+#define TK_IMMEDIATE                      322
+#define TK_IMPORT                         323
+#define TK_INITIALLY                      324
+#define TK_INSTEAD                        325
+#define TK_ISNULL                         326
+#define TK_KEY                            327
+#define TK_MODULES                        328
+#define TK_NK_BITNOT                      329
+#define TK_NK_SEMI                        330
+#define TK_NOTNULL                        331
+#define TK_OF                             332
+#define TK_PLUS                           333
+#define TK_PRIVILEGE                      334
+#define TK_RAISE                          335
+#define TK_RESTRICT                       336
+#define TK_ROW                            337
+#define TK_SEMI                           338
+#define TK_STAR                           339
+#define TK_STATEMENT                      340
+#define TK_STRICT                         341
+#define TK_STRING                         342
+#define TK_TIMES                          343
+#define TK_VALUES                         344
+#define TK_VARIABLE                       345
+#define TK_WAL                            346
 
 #define TK_NK_SPACE         600
 #define TK_NK_COMMENT       601

include/common/ttypes.h

@@ -275,9 +275,11 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT64(_t)     ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
+#define IS_VALID_UINT64(_t)    ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -37,6 +37,10 @@ typedef struct SVariant {
   };
 } SVariant;
 
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value);
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value);
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 

include/dnode/snode/snode.h

@@ -45,6 +45,7 @@ typedef struct {
  */
 SSnode *sndOpen(const char *path, const SSnodeOpt *pOption);
 
+int32_t sndInit(SSnode * pSnode);
 /**
  * @brief Stop Snode in Dnode.
  *

include/dnode/vnode/tqCommon.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TQ_COMMON_H
+#define TDENGINE_TQ_COMMON_H
+
+// message process
+int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart);
+int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored);
+int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
+int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored);
+int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
+int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
+int32_t startStreamTasks(SStreamMeta* pMeta);
+int32_t resetStreamTaskStatus(SStreamMeta* pMeta);
+
+#endif  // TDENGINE_TQ_COMMON_H

include/libs/executor/storageapi.h

@@ -38,8 +38,8 @@ extern "C" {
 
 #define META_READER_NOLOCK 0x1
 
-#define  STREAM_STATE_BUFF_HASH 1
-#define  STREAM_STATE_BUFF_SORT 2
+#define STREAM_STATE_BUFF_HASH 1
+#define STREAM_STATE_BUFF_SORT 2
 
 typedef struct SMeta SMeta;
 typedef TSKEY (*GetTsFun)(void*);
@@ -102,14 +102,14 @@ typedef struct SMTbCursor {
 } SMTbCursor;
 
 typedef struct SMCtbCursor {
-  SMeta   *pMeta;
-  void    *pCur;
+  SMeta*   pMeta;
+  void*    pCur;
   tb_uid_t suid;
-  void    *pKey;
-  void    *pVal;
+  void*    pKey;
+  void*    pVal;
   int      kLen;
   int      vLen;
-  int8_t     paused;
+  int8_t   paused;
   int      lock;
 } SMCtbCursor;
 
@@ -152,10 +152,22 @@ typedef struct {
 
 // clang-format off
 /*-------------------------------------------------new api format---------------------------------------------------*/
+typedef enum {
+  TSD_READER_NOTIFY_DURATION_START,
+  TSD_READER_NOTIFY_NEXT_DURATION_BLOCK,
+} ETsdReaderNotifyType;
+
+typedef union {
+  struct {
+    int32_t filesetId;
+  } duration;
+} STsdReaderNotifyInfo;
+
+typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param);
+
 typedef struct TsdReader {
   int32_t      (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
-                           SHashObj** pIgnoreTables);
+                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
   void         (*tsdReaderClose)();
   void         (*tsdSetReaderTaskId)(void *pReader, const char *pId);
   int32_t      (*tsdSetQueryTableList)();
@@ -170,6 +182,9 @@ typedef struct TsdReader {
   int32_t      (*tsdReaderGetDataBlockDistInfo)();
   int64_t      (*tsdReaderGetNumOfInMemRows)();
   void         (*tsdReaderNotifyClosing)();
+
+  void         (*tsdSetFilesetDelimited)(void* pReader);
+  void         (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 } TsdReader;
 
 typedef struct SStoreCacheReader {
@@ -248,22 +263,23 @@ typedef struct SStoreMeta {
   void* (*storeGetIndexInfo)();
   void* (*getInvertIndex)(void* pVnode);
   // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
-  int32_t (*getChildTableList)( void* pVnode, int64_t suid, SArray* list);
+  int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
   int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);
   void* storeGetVersionRange;
   void* storeGetLastTimestamp;
 
   int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid);  // tsdbGetTableSchema
-  int32_t (*getNumOfChildTables)( void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
-  void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
+  int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
+  void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
+                       int64_t* numOfNormalTables);
 
   int64_t (*getNumOfRowsInMem)(void* pVnode);
 
-  SMCtbCursor* (*openCtbCursor)(void *pVnode, tb_uid_t uid, int lock);
-  int32_t      (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
-  void         (*pauseCtbCursor)(SMCtbCursor* pCtbCur);
-  void         (*closeCtbCursor)(SMCtbCursor *pCtbCur);
-  tb_uid_t     (*ctbCursorNext)(SMCtbCursor* pCur);
+  SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
+  int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
+  void (*pauseCtbCursor)(SMCtbCursor* pCtbCur);
+  void (*closeCtbCursor)(SMCtbCursor* pCtbCur);
+  tb_uid_t (*ctbCursorNext)(SMCtbCursor* pCur);
 } SStoreMeta;
 
 typedef struct SStoreMetaReader {
@@ -348,12 +364,14 @@ typedef struct SStateStore {
                                                           const SSessionKey* pKey, void** pVal, int32_t* pVLen);
 
   SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp);
-  TSKEY        (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
-  bool         (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
-  bool         (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
+  TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
+  bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
+  bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
+  bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
+
   void (*updateInfoDestroy)(SUpdateInfo* pInfo);
-  void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count);
-  void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count);
+  void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
+  void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
 
   SUpdateInfo* (*updateInfoInitP)(SInterval* pInterval, int64_t watermark, bool igUp);
   void (*updateInfoAddCloseWindowSBF)(SUpdateInfo* pInfo);
@@ -380,6 +398,7 @@ typedef struct SStateStore {
   void (*streamStateDestroy)(SStreamState* pState, bool remove);
   int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
   void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
+  void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
 } SStateStore;
 
 typedef struct SStorageAPI {

include/libs/function/function.h

@@ -171,6 +171,7 @@ typedef struct {
   int32_t                  taskId;
   int64_t                  streamId;
   int64_t                  streamBackendRid;
+  int8_t                   dump;
 } SStreamState;
 
 typedef struct SFunctionStateStore {

include/libs/nodes/cmdnodes.h

@@ -46,6 +46,10 @@ extern "C" {
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
 
+#define COMPACT_DB_RESULT_COLS 3
+#define COMPACT_DB_RESULT_FIELD1_LEN 32
+#define COMPACT_DB_RESULT_FIELD3_LEN 128
+
 #define SHOW_ALIVE_RESULT_COLS 1
 
 #define BIT_FLAG_MASK(n)              (1 << n)
@@ -335,6 +339,15 @@ typedef struct SShowTableTagsStmt {
   SNodeList* pTags;
 } SShowTableTagsStmt;
 
+typedef struct SShowCompactsStmt {
+  ENodeType type;
+} SShowCompactsStmt;
+
+typedef struct SShowCompactDetailsStmt {
+  ENodeType type;
+  SNode* pCompactId;
+} SShowCompactDetailsStmt;
+
 typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
 
 typedef struct SIndexOptions {

include/libs/nodes/plannodes.h

@@ -118,6 +118,7 @@ typedef struct SScanLogicNode {
   bool          igLastNull;
   bool          groupOrderScan;
   bool          onlyMetaCtbIdx; // for tag scan with no tbname
+  bool          filesetDelimited; // returned blocks delimited by fileset
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -432,6 +433,7 @@ typedef struct STableScanPhysiNode {
   int8_t         igExpired;
   bool           assignBlockUid;
   int8_t         igCheckUpdate;
+  bool           filesetDelimited;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;

include/libs/nodes/querynodes.h

@@ -449,6 +449,7 @@ typedef struct SVnodeModifyOpStmt {
   SHashObj*             pSubTableHashObj;    // SHashObj<table_name, STableMeta*>
   SHashObj*             pTableNameHashObj;   // set of table names for refreshing meta, sync mode
   SHashObj*             pDbFNameHashObj;     // set of db names for refreshing meta, sync mode
+  SHashObj*             pTableCxtHashObj;    // temp SHashObj<tuid, STableDataCxt*> for single request
   SArray*               pVgDataBlocks;  // SArray<SVgroupDataCxt*>
   SVCreateTbReq*        pCreateTblReq;
   TdFilePtr             fp;

include/libs/stream/streamState.h

@@ -35,6 +35,7 @@ int32_t       streamStateBegin(SStreamState* pState);
 int32_t       streamStateCommit(SStreamState* pState);
 void          streamStateDestroy(SStreamState* pState, bool remove);
 int32_t       streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
+int32_t       streamStateDelTaskDb(SStreamState* pState);
 
 int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
@@ -49,7 +50,7 @@ void    streamStateSetNumber(SStreamState* pState, int32_t number);
 int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, void* pVal, int32_t vLen);
 int32_t streamStateGetInfo(SStreamState* pState, void* pKey, int32_t keyLen, void** pVal, int32_t* pLen);
 
-//session window
+// session window
 int32_t streamStateSessionAddIfNotExist(SStreamState* pState, SSessionKey* key, TSKEY gap, void** pVal, int32_t* pVLen);
 int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void* value, int32_t vLen);
 int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen);
@@ -64,7 +65,7 @@ SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSess
 SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key);
 SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, const SSessionKey* key);
 
-//state window
+// state window
 int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen,
                                       state_key_cmpr_fn fn, void** pVal, int32_t* pVLen);
 
@@ -95,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
 int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
 
 void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
+
+void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
+
 SStreamStateCur* createStreamStateCursor();
 
 /***compare func **/
@@ -133,4 +137,4 @@ char* streamStateIntervalDump(SStreamState* pState);
 }
 #endif
 
-#endif /* ifndef _STREAM_STATE_H_ */
+#endif /* ifndef _STREAM_STATE_H_ */

include/libs/stream/tstream.h

@@ -34,28 +34,33 @@ extern "C" {
 #define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F)
 #define SIZE_IN_KiB(_v) ((_v) / ONE_KiB_F)
 
+#define TASK_DOWNSTREAM_READY      0x0
+#define TASK_DOWNSTREAM_NOT_READY  0x1
+#define TASK_DOWNSTREAM_NOT_LEADER 0x2
+#define TASK_UPSTREAM_NEW_STAGE    0x3
 
-#define TASK_DOWNSTREAM_READY       0x0
-#define TASK_DOWNSTREAM_NOT_READY   0x1
-#define TASK_DOWNSTREAM_NOT_LEADER  0x2
-#define TASK_UPSTREAM_NEW_STAGE     0x3
+#define NODE_ROLE_UNINIT   0x1
+#define NODE_ROLE_LEADER   0x2
+#define NODE_ROLE_FOLLOWER 0x3
 
-#define NODE_ROLE_UNINIT     0x1
-#define NODE_ROLE_LEADER     0x2
-#define NODE_ROLE_FOLLOWER   0x3
-
-#define HAS_RELATED_FILLHISTORY_TASK(_t)  ((_t)->hTaskInfo.id.taskId != 0)
+#define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0)
 #define CLEAR_RELATED_FILLHISTORY_TASK(_t) \
   do {                                     \
     (_t)->hTaskInfo.id.taskId = 0;         \
     (_t)->hTaskInfo.id.streamId = 0;       \
   } while (0)
 
+#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1)
+#define STREAM_EXEC_START_ALL_TASKS_ID     (-2)
+#define STREAM_EXEC_RESTART_ALL_TASKS_ID   (-3)
+
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
 typedef struct SStreamTaskSM SStreamTaskSM;
 
-#define SSTREAM_TASK_VER 2
+#define SSTREAM_TASK_VER              2
+#define SSTREAM_TASK_INCOMPATIBLE_VER 1
+#define SSTREAM_TASK_NEED_CONVERT_VER 2
 
 enum {
   STREAM_STATUS__NORMAL = 0,
@@ -107,6 +112,7 @@ typedef enum {
   TASK_LEVEL__SOURCE = 1,
   TASK_LEVEL__AGG,
   TASK_LEVEL__SINK,
+  TASK_LEVEL_SMA,
 } ETASK_LEVEL;
 
 enum {
@@ -159,7 +165,7 @@ typedef struct {
 typedef struct {
   int8_t  type;
   int64_t ver;
-  SArray* submits;   // SArray<SPackedSubmit>
+  SArray* submits;  // SArray<SPackedSubmit>
 } SStreamMergedSubmit;
 
 typedef struct {
@@ -210,9 +216,6 @@ int32_t         streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem);
 SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue);
 #endif
 
-int32_t streamInit();
-void    streamCleanUp();
-
 SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type);
 void               streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
 
@@ -251,7 +254,7 @@ typedef struct {
 } SScanhistoryDataInfo;
 
 typedef struct {
-  int32_t idleDuration;   // idle time before use time slice the continue execute scan-history
+  int32_t idleDuration;  // idle time before use time slice the continue execute scan-history
   int32_t numOfTicks;
   tmr_h   pTimer;
   int32_t execCount;
@@ -301,11 +304,16 @@ typedef struct SStreamTaskId {
 typedef struct SCheckpointInfo {
   int64_t startTs;
   int64_t checkpointId;
-  int64_t checkpointVer;      // latest checkpointId version
-  int64_t processedVer;        // already processed ver, that has generated results version.
-  int64_t nextProcessVer;     // current offset in WAL, not serialize it
-  int64_t failedId;           // record the latest failed checkpoint id
+
+  int64_t checkpointVer;  // latest checkpointId version
+  int64_t processedVer;
+  int64_t nextProcessVer;  // current offset in WAL, not serialize it
+  int64_t failedId;        // record the latest failed checkpoint id
+  int64_t checkpointingId;
+  int32_t downstreamAlignNum;
+  int32_t checkpointNotReadyTasks;
   bool    dispatchCheckpointTrigger;
+  int64_t msgVer;
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
@@ -331,22 +339,22 @@ typedef struct SSTaskBasicInfo {
   int32_t selfChildId;
   int32_t totalLevel;
   int8_t  taskLevel;
-  int8_t  fillHistory;  // is fill history task or not
-  int64_t triggerParam; // in msec
+  int8_t  fillHistory;   // is fill history task or not
+  int64_t triggerParam;  // in msec
 } SSTaskBasicInfo;
 
 typedef struct SStreamDispatchReq SStreamDispatchReq;
-typedef struct STokenBucket STokenBucket;
-typedef struct SMetaHbInfo  SMetaHbInfo;
+typedef struct STokenBucket       STokenBucket;
+typedef struct SMetaHbInfo        SMetaHbInfo;
 
 typedef struct SDispatchMsgInfo {
-  SStreamDispatchReq*   pData;       // current dispatch data
-  int8_t  dispatchMsgType;
-  int16_t msgType;     // dispatch msg type
-  int32_t retryCount;  // retry send data count
-  int64_t startTs;     // dispatch start time, record total elapsed time for dispatch
-  SArray* pRetryList;  // current dispatch successfully completed node of downstream
-  void*   pTimer; // used to dispatch data after a given time duration
+  SStreamDispatchReq* pData;  // current dispatch data
+  int8_t              dispatchMsgType;
+  int16_t             msgType;     // dispatch msg type
+  int32_t             retryCount;  // retry send data count
+  int64_t             startTs;     // dispatch start time, record total elapsed time for dispatch
+  SArray*             pRetryList;  // current dispatch successfully completed node of downstream
+  void*               pTimer;      // used to dispatch data after a given time duration
 } SDispatchMsgInfo;
 
 typedef struct STaskQueue {
@@ -355,8 +363,8 @@ typedef struct STaskQueue {
 } STaskQueue;
 
 typedef struct STaskSchedInfo {
-  int8_t  status;
-  void*   pTimer;
+  int8_t status;
+  void*  pTimer;
 } STaskSchedInfo;
 
 typedef struct SSinkRecorder {
@@ -444,12 +452,11 @@ struct SStreamTask {
   int64_t             checkReqId;
   SArray*             checkReqIds;  // shuffle
   int32_t             refCnt;
-  int64_t             checkpointingId;
-  int32_t             checkpointAlignCnt;
-  int32_t             checkpointNotReadyTasks;
   int32_t             transferStateAlignCnt;
   struct SStreamMeta* pMeta;
   SSHashObj*          pNameMap;
+  void*               pBackend;
+  int64_t             backendRefId;
   char                reserve[256];
 };
 
@@ -457,9 +464,9 @@ typedef struct STaskStartInfo {
   int64_t   startTs;
   int64_t   readyTs;
   int32_t   tasksWillRestart;
-  int32_t   taskStarting;          // restart flag, sentinel to guard the restart procedure.
-  SHashObj* pReadyTaskSet;         // tasks that are all ready for running stream processing
-  SHashObj* pFailedTaskSet;        // tasks that are done the check downstream process, may be successful or failed
+  int32_t   taskStarting;    // restart flag, sentinel to guard the restart procedure.
+  SHashObj* pReadyTaskSet;   // tasks that are all ready for running stream processing
+  SHashObj* pFailedTaskSet;  // tasks that are done the check downstream process, may be successful or failed
   int64_t   elapsedTime;
 } STaskStartInfo;
 
@@ -487,20 +494,22 @@ typedef struct SStreamMeta {
   int32_t         walScanCounter;
   void*           streamBackend;
   int64_t         streamBackendRid;
-  SHashObj*       pTaskBackendUnique;
+  SHashObj*       pTaskDbUnique;
   TdThreadMutex   backendMutex;
   SMetaHbInfo*    pHbInfo;
   STaskUpdateInfo updateInfo;
+  SHashObj*       pUpdateTaskSet;
   int32_t         numOfStreamTasks;  // this value should be increased when a new task is added into the meta
   int32_t         numOfPausedTasks;
-  int32_t         chkptNotReadyTasks;
   int64_t         rid;
 
   int64_t  chkpId;
+  int32_t  chkpCap;
   SArray*  chkpSaved;
   SArray*  chkpInUse;
-  int32_t  chkpCap;
   SRWLatch chkpDirLock;
+  void*    qHandle;
+  void*    bkdChkptMgt;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -532,7 +541,7 @@ struct SStreamDispatchReq {
   int64_t stage;  // nodeId from upstream task
   int64_t streamId;
   int32_t taskId;
-  int32_t msgId;      // msg id to identify if the incoming msg from the same sender
+  int32_t msgId;  // msg id to identify if the incoming msg from the same sender
   int32_t srcVgId;
   int32_t upstreamTaskId;
   int32_t upstreamChildId;
@@ -591,7 +600,7 @@ typedef struct {
   int32_t downstreamNodeId;
   int32_t downstreamTaskId;
   int32_t childId;
-  int32_t oldStage;
+  int64_t oldStage;
   int8_t  status;
 } SStreamTaskCheckRsp;
 
@@ -656,7 +665,7 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea
 typedef struct STaskStatusEntry {
   STaskId id;
   int32_t status;
-  int32_t statusLastDuration; // to record the last duration of current status
+  int32_t statusLastDuration;  // to record the last duration of current status
   int64_t stage;
   int32_t nodeId;
   int64_t verStart;         // start version in WAL, only valid for source task
@@ -669,15 +678,15 @@ typedef struct STaskStatusEntry {
   int64_t inputQUnchangeCounter;
   double  inputQUsed;       // in MiB
   double  inputRate;
-  double  sinkQuota;        // existed quota size for sink task
-  double  sinkDataSize;     // sink to dst data size
+  double  sinkQuota;     // existed quota size for sink task
+  double  sinkDataSize;  // sink to dst data size
 } STaskStatusEntry;
 
 typedef struct SStreamHbMsg {
   int32_t vgId;
   int32_t numOfTasks;
-  SArray* pTaskStatus;  // SArray<STaskStatusEntry>
-  SArray* pUpdateNodes; // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
+  SArray* pTaskStatus;   // SArray<STaskStatusEntry>
+  SArray* pUpdateNodes;  // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
 } SStreamHbMsg;
 
 int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
@@ -702,7 +711,7 @@ typedef struct SNodeUpdateInfo {
 } SNodeUpdateInfo;
 
 typedef struct SStreamTaskNodeUpdateMsg {
-  int32_t transId;   // to identify the msg
+  int32_t transId;  // to identify the msg
   int64_t streamId;
   int32_t taskId;
   SArray* pNodeList;  // SArray<SNodeUpdateInfo>
@@ -759,12 +768,13 @@ const char* streamTaskGetStatusStr(ETaskStatus status);
 void        streamTaskResetStatus(SStreamTask* pTask);
 void        streamTaskSetStatusReady(SStreamTask* pTask);
 
-void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
+void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
 
 // recover and fill history
-void    streamTaskCheckDownstream(SStreamTask* pTask);
+void streamTaskCheckDownstream(SStreamTask* pTask);
 
-int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
+int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
+                              int64_t* oldStage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
 void    streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
 bool    streamTaskAllUpstreamClosed(SStreamTask* pTask);
@@ -790,17 +800,17 @@ bool    streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer)
 int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
 
 // common
-int32_t     streamRestoreParam(SStreamTask* pTask);
-void        streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
-void        streamTaskResume(SStreamTask* pTask);
-int32_t     streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
-void        streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
-void        streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
-void        streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask);
-int32_t     streamTaskReleaseState(SStreamTask* pTask);
-int32_t     streamTaskReloadState(SStreamTask* pTask);
-void        streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
-void        streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
+int32_t streamRestoreParam(SStreamTask* pTask);
+void    streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
+void    streamTaskResume(SStreamTask* pTask);
+int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
+void    streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
+void    streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
+void    streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask);
+int32_t streamTaskReleaseState(SStreamTask* pTask);
+int32_t streamTaskReloadState(SStreamTask* pTask);
+void    streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId);
+void    streamTaskOpenAllUpstreamInput(SStreamTask* pTask);
 
 void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
 void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);
@@ -809,7 +819,7 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);
 int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow);
 int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow);
 SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
-int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask);
+int32_t              streamDispatchScanHistoryFinishMsg(SStreamTask* pTask);
 
 // agg level
 int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, SRpcHandleInfo* pInfo);
@@ -827,14 +837,14 @@ int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int3
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
-int32_t      streamMetaReopen(SStreamMeta* pMeta);
+void         streamMetaClear(SStreamMeta* pMeta);
 void         streamMetaInitBackend(SStreamMeta* pMeta);
 int32_t      streamMetaCommit(SStreamMeta* pMeta);
 int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
 int64_t      streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
+int32_t      streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);
 void         streamMetaStartHb(SStreamMeta* pMeta);
-void         streamMetaInitForSnode(SStreamMeta* pMeta);
 bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
 int32_t      streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
                                                   int64_t endTs, bool ready);
@@ -848,7 +858,7 @@ void         streamMetaResetStartInfo(STaskStartInfo* pMeta);
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
-void    streamTaskClearCheckInfo(SStreamTask* pTask);
+void    streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,
@@ -856,8 +866,10 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
 int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
                                  int8_t isSucceed);
 
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
+void*          streamDestroyStateMachine(SStreamTaskSM* pSM);
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* ifndef _STREAM_H_ */
+#endif /* ifndef _STREAM_H_ */

include/libs/stream/tstreamUpdate.h

@@ -55,6 +55,7 @@ int32_t      updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *p
 int32_t      updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
 void         windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
 void         windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
+bool         isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
 
 #ifdef __cplusplus
 }

include/libs/sync/sync.h

@@ -33,11 +33,12 @@ extern "C" {
 #define SYNC_MAX_PROGRESS_WAIT_MS    4000
 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
 #define SYNC_MAX_RECV_TIME_RANGE_MS  1200
-#define SYNC_DEL_WAL_MS              (1000 * 60)
 #define SYNC_ADD_QUORUM_COUNT        3
 #define SYNC_VNODE_LOG_RETENTION     (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
 #define SNAPSHOT_WAIT_MS             1000 * 5
 
+#define SYNC_WAL_LOG_RETENTION_SIZE (8LL * 1024 * 1024 * 1024)
+
 #define SYNC_MAX_RETRY_BACKOFF         5
 #define SYNC_LOG_REPL_RETRY_WAIT_MS    100
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
@@ -45,8 +46,8 @@ extern "C" {
 
 #define SYNC_HEARTBEAT_SLOW_MS       1500
 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
-#define SYNC_SNAP_RESEND_MS          1000 * 60
-#define SYNC_SNAP_TIMEOUT_MS         1000 * 600
+#define SYNC_SNAP_RESEND_MS          1000 * 300
+#define SYNC_SNAP_TIMEOUT_MS         1000 * 1800
 
 #define SYNC_VND_COMMIT_MIN_MS 3000
 
@@ -219,6 +220,7 @@ typedef struct SSyncLogStore {
 
   SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
   SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogIndexRetention)(struct SSyncLogStore* pLogStore, int64_t bytes);
   SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
 
   int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);
@@ -289,6 +291,9 @@ const char* syncStr(ESyncState state);
 
 int32_t    syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
 
+// util
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/wal/wal.h

@@ -225,6 +225,7 @@ bool walIsEmpty(SWal *);
 int64_t walGetFirstVer(SWal *);
 int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);
+int64_t walGetVerRetention(SWal *pWal, int64_t bytes);
 int64_t walGetCommittedVer(SWal *);
 int64_t walGetAppliedVer(SWal *);
 

include/os/osFile.h

@@ -54,15 +54,18 @@ extern "C" {
 
 typedef struct TdFile *TdFilePtr;
 
-#define TD_FILE_CREATE   0x0001
-#define TD_FILE_WRITE    0x0002
-#define TD_FILE_READ     0x0004
-#define TD_FILE_TRUNC    0x0008
-#define TD_FILE_APPEND   0x0010
-#define TD_FILE_TEXT     0x0020
-#define TD_FILE_AUTO_DEL 0x0040
-#define TD_FILE_EXCL     0x0080
-#define TD_FILE_STREAM   0x0100  // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
+#define TD_FILE_CREATE        0x0001
+#define TD_FILE_WRITE         0x0002
+#define TD_FILE_READ          0x0004
+#define TD_FILE_TRUNC         0x0008
+#define TD_FILE_APPEND        0x0010
+#define TD_FILE_TEXT          0x0020
+#define TD_FILE_AUTO_DEL      0x0040
+#define TD_FILE_EXCL          0x0080
+#define TD_FILE_STREAM        0x0100  // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
+#define TD_FILE_WRITE_THROUGH 0x0200
+#define TD_FILE_CLOEXEC       0x0400
+
 TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions);
 TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions);
 

include/os/osString.h

@@ -47,18 +47,13 @@ typedef int32_t TdUcs4;
 #define strtof      STR_TO_F_FUNC_TAOS_FORBID
 #endif
 
-#ifdef WINDOWS
-#define tstrdup(str) _strdup(str)
-#else
-#define tstrdup(str) strdup(str)
-#endif
-
 #define tstrncpy(dst, src, size)   \
   do {                             \
     strncpy((dst), (src), (size)); \
     (dst)[(size)-1] = 0;           \
   } while (0)
 
+char   *tstrdup(const char *src);
 int32_t taosUcs4len(TdUcs4 *ucs4);
 int64_t taosStr2int64(const char *str);
 

include/util/taoserror.h

@@ -124,6 +124,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_CFG_VALUE             TAOS_DEF_ERROR_CODE(0, 0x0133)
 
 #define TSDB_CODE_IP_NOT_IN_WHITE_LIST          TAOS_DEF_ERROR_CODE(0, 0x0134)
+#define TSDB_CODE_FAILED_TO_CONNECT_S3          TAOS_DEF_ERROR_CODE(0, 0x0135)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)
@@ -426,6 +427,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_VIEW_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x04A0)
 #define TSDB_CODE_MND_VIEW_NOT_EXIST            TAOS_DEF_ERROR_CODE(0, 0x04A1)
 
+//mnode-compact
+#define TSDB_CODE_MND_INVALID_COMPACT_ID        TAOS_DEF_ERROR_CODE(0, 0x04B1)
+
 
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

include/util/tdef.h

@@ -288,6 +288,7 @@ typedef enum ELogicConditionType {
 #define TSDB_CONN_ACTIVE_KEY_LEN 255
 
 #define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024)
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120  // default payload size, greater than PATH_MAX value
@@ -305,7 +306,7 @@ typedef enum ELogicConditionType {
 #define TSDB_SYNC_APPLYQ_SIZE_LIMIT    512
 #define TSDB_SYNC_NEGOTIATION_WIN      512
 
-#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048
+#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024
 
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta

include/util/tlog.h

@@ -66,6 +66,7 @@ extern int32_t udfDebugFlag;
 extern int32_t smaDebugFlag;
 extern int32_t idxDebugFlag;
 extern int32_t tdbDebugFlag;
+extern int32_t sndDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

source/client/inc/clientInt.h

@@ -205,7 +205,7 @@ typedef struct SRequestSendRecvBody {
   __taos_async_fn_t queryFp;
   __taos_async_fn_t fetchFp;
   EQueryExecMode    execMode;
-  void*             param;
+  void*             interParam;
   SDataBuf          requestMsg;
   int64_t           queryJob;  // query job, created according to sql query DAG.
   int32_t           subplanNum;
@@ -287,6 +287,7 @@ typedef struct SRequestObj {
 typedef struct SSyncQueryParam {
   tsem_t       sem;
   SRequestObj* pRequest;
+  void*        userParam;
 } SSyncQueryParam;
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
@@ -420,6 +421,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
 int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
 void    returnToUser(SRequestObj* pRequest);
 void    stopAllQueries(SRequestObj *pRequest);
+void    doRequestCallback(SRequestObj* pRequest, int32_t code);
 void    freeQueryParam(SSyncQueryParam* param);
 
 #ifdef TD_ENTERPRISE

source/client/inc/clientSml.h

@@ -65,14 +65,32 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-//#define TS        "_ts"
-//#define TS_LEN    3
 #define VALUE     "_value"
-#define VALUE_LEN 6
+#define VALUE_LEN (sizeof(VALUE)-1)
 
 #define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 10
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
+
+#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+
+#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
+
+#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
+
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
 
 typedef enum {
   SCHEMA_ACTION_NULL,
@@ -83,18 +101,6 @@ typedef enum {
   SCHEMA_ACTION_CHANGE_TAG_SIZE,
 } ESchemaAction;
 
-typedef struct {
-  const void  *key;
-  int32_t      keyLen;
-  void        *value;
-  bool         used;
-}Node;
-
-typedef struct NodeList{
-  Node             data;
-  struct NodeList* next;
-}NodeList;
-
 typedef struct {
   char *measure;
   char *tags;
@@ -117,7 +123,6 @@ typedef struct {
   int32_t     sTableNameLen;
   char        childTableName[TSDB_TABLE_NAME_LEN];
   uint64_t    uid;
-//  void       *key;        // for openTsdb
 
   SArray *tags;
 
@@ -161,7 +166,8 @@ typedef struct {
 typedef struct {
   int64_t id;
 
-  SMLProtocolType protocol;
+  TSDB_SML_PROTOCOL_TYPE protocol;
+
   int8_t          precision;
   bool            reRun;
   bool            dataFormat;  // true means that the name and order of keys in each line are the same(only for influx protocol)
@@ -201,29 +207,8 @@ typedef struct {
   bool         needModifySchema;
 } SSmlHandle;
 
-#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
-
-#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
-
-#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
-
-#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
-
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
-
-#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
-  for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
-      MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      keyLen--;                              \
-    }                                        \
-  }
-
-extern int64_t smlFactorNS[3];
-extern int64_t smlFactorS[3];
+extern int64_t smlFactorNS[];
+extern int64_t smlFactorS[];
 
 typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
@@ -231,16 +216,10 @@ SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
 int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-//int           nodeListSize(NodeList* list);
-bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
 int64_t       smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
-int8_t        smlGetTsTypeByLen(int32_t len);
 SSmlTableInfo*    smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
 SSmlSTableMeta*   smlBuildSTableMeta(bool isDataFormat);
 int32_t           smlSetCTableName(SSmlTableInfo *oneTable);
@@ -253,12 +232,45 @@ int32_t           smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg);
 uint8_t           smlGetTimestampLen(int64_t num);
 void              smlDestroyTableInfo(void *para);
 
-void freeSSmlKv(void* data);
+void    freeSSmlKv(void* data);
 int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
-void    smlStrReplace(char* src, int32_t len);
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
+bool            isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+bool            isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+int32_t         smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlJoinMeasureTag(SSmlLineInfo *elements);
+void            smlBuildTsKv(SSmlKv *kv, int64_t ts);
+int32_t         smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
+int32_t         smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
+
+static inline bool smlDoubleToInt64OverFlow(double num) {
+  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
+  return false;
+}
+
+static inline void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
+static inline int8_t smlGetTsTypeByLen(int32_t len) {
+  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
+    return TSDB_TIME_PRECISION_SECONDS;
+  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+    return TSDB_TIME_PRECISION_MILLI;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -316,6 +316,15 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
+  SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (interParam == NULL) {
+    doDestroyRequest(pRequest);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  tsem_init(&interParam->sem, 0, 0);
+  interParam->pRequest = pRequest;
+  pRequest->body.interParam = interParam;
 
   pRequest->resType = RES_TYPE__QUERY;
   pRequest->requestId = reqid == 0 ? generateRequestId() : reqid;
@@ -437,12 +446,10 @@ void doDestroyRequest(void *p) {
     deregisterRequest(pRequest);
   }
 
-  if (pRequest->syncQuery) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
+  if (pRequest->body.interParam) {
+    tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
   }
+  taosMemoryFree(pRequest->body.interParam);
 
   qDestroyQuery(pRequest->pQuery);
   nodesDestroyAllocator(pRequest->allocatorRefId);
@@ -555,6 +562,9 @@ static void *tscCrashReportThreadFp(void *param) {
         if (pFile) {
           taosReleaseCrashLogFile(pFile, false);
           pFile = NULL;
+
+          taosMsleep(sleepTime);
+          loopTimes = 0;
           continue;
         }
       } else {
@@ -729,7 +739,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         return -1;
       }
       newstr[0] = '"';
-      strncpy(newstr+1, str, len);
+      memcpy(newstr+1, str, len);
       newstr[len + 1] = '"';
       newstr[len + 2] = '\0';
       str = newstr;

source/client/src/clientImpl.c

@@ -196,21 +196,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   (*pRequest)->sqlLen = sqlLen;
   (*pRequest)->validateOnly = validateSql;
 
-  SSyncQueryParam* newpParam = NULL;
-  if (param == NULL) {
-    newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-    if (newpParam == NULL) {
-      destroyRequest(*pRequest);
-      *pRequest = NULL;
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    tsem_init(&newpParam->sem, 0, 0);
-    newpParam->pRequest = (*pRequest);
-    param = newpParam;
-  }
-
-  (*pRequest)->body.param = param;
+  ((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
 
   STscObj* pTscObj = (*pRequest)->pTscObj;
   int32_t  err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
@@ -218,7 +204,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   if (err) {
     tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
              (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-    freeQueryParam(newpParam);
     destroyRequest(*pRequest);
     *pRequest = NULL;
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -230,7 +215,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
         nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
       tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
                (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-      freeQueryParam(newpParam);
       destroyRequest(*pRequest);
       *pRequest = NULL;
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -336,7 +320,7 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb
 void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   SRetrieveTableRsp* pRsp = NULL;
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -358,18 +342,18 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
              pRequest->requestId);
   }
 
-  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+  doRequestCallback(pRequest, code);
 }
 
 int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
   // drop table if exists not_exists_table
   if (NULL == pQuery->pCmdMsg) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -384,7 +368,7 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   int64_t transporterId = 0;
   int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
   if (code) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
   return code;
 }
@@ -913,7 +897,7 @@ void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
 void returnToUser(SRequestObj* pRequest) {
   if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) {
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
     return;
   }
 
@@ -921,7 +905,7 @@ void returnToUser(SRequestObj* pRequest) {
   if (pUserReq) {
     pUserReq->code = pRequest->code;
     // return to client
-    pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code);
+    doRequestCallback(pUserReq, pUserReq->code);
     releaseRequest(pRequest->relation.userRefId);
     return;
   } else {
@@ -1031,7 +1015,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
     pRequest->pWrapper = NULL;
 
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1186,7 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
       pRequest->code = terrno;
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 
   // todo not to be released here
@@ -1199,7 +1183,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
   int32_t code = 0;
 
   if (pRequest->parseOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -1233,11 +1217,11 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
       pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
-      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+      doRequestCallback(pRequest, 0);
       break;
     default:
       tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
-      pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
+      doRequestCallback(pRequest, -1);
       break;
   }
 }
@@ -1703,8 +1687,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
 }
 
 static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
-  SSyncQueryParam* pParam = param;
-  tsem_post(&pParam->sem);
+  tsem_t* sem = param;
+  tsem_post(sem);
 }
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
@@ -1722,10 +1706,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
 
     // convert ucs4 to native multi-bytes string
     pResultInfo->convertUcs4 = convertUcs4;
-
-    SSyncQueryParam* pParam = pRequest->body.param;
-    taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
-    tsem_wait(&pParam->sem);
+    tsem_t sem;
+    tsem_init(&sem, 0, 0);
+    taos_fetch_rows_a(pRequest, syncFetchFn, &sem);
+    tsem_wait(&sem);
+    tsem_destroy(&sem);
   }
 
   if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@@ -2492,6 +2477,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   tscDebug("taos_query start with sql:%s", sql);
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (NULL == param) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
@@ -2501,9 +2490,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
+  taosMemoryFree(param);
 
   tscDebug("taos_query end with sql:%s", sql);
 
@@ -2517,19 +2505,20 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
   }
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (param == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
   tsem_wait(&param->sem);
-
   SRequestObj* pRequest = NULL;
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
-
+  taosMemoryFree(param);
   return pRequest;
 }
 
@@ -2547,13 +2536,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     pRequest->code = code;
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
@@ -2574,20 +2563,12 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
     atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
   }
 
-  pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
+  pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, pResultInfo->numOfRows);
 }
 
 void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
-  if (pRequest->syncQuery && pRequest->body.param != param) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
-    pRequest->syncQuery = false;
-  }
-  
   pRequest->body.fetchFp = fp;
-  pRequest->body.param = param;
+  ((SSyncQueryParam *)pRequest->body.interParam)->userParam = param;
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
 
@@ -2625,6 +2606,10 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
   schedulerFetchRows(pRequest->body.queryJob, &req);
 }
 
+void doRequestCallback(SRequestObj* pRequest, int32_t code) {
+  pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
+}
+
 int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
 #ifndef TD_ENTERPRISE
   return TSDB_CODE_SUCCESS;
@@ -2633,4 +2618,3 @@ int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool pa
 #endif
 }
 
-

source/client/src/clientMain.c

@@ -876,6 +876,7 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
     code = 0;
   }
   taosThreadMutexUnlock(&pTscObj->mutex);
+  releaseTscObj(*(int64_t *)taos);
   return code;
 }
 
@@ -1094,7 +1095,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1111,7 +1112,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1209,7 +1210,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
     terrno = code;
     pRequest->code = code;
     tscDebug("call sync query cb with code: %s", tstrerror(code));
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
     return;
   }
 
@@ -1241,7 +1242,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1548,12 +1549,12 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
 
   conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
-  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.param, NULL);
+  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.interParam, NULL);
   if (code) {
     goto _return;
   }
 
-  SSyncQueryParam *pParam = pRequest->body.param;
+  SSyncQueryParam *pParam = pRequest->body.interParam;
   tsem_wait(&pParam->sem);
 
 _return:

source/client/src/clientMsgHandler.c

@@ -16,6 +16,7 @@
 #include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "cmdnodes.h"
 #include "os.h"
 #include "query.h"
 #include "systable.h"
@@ -41,7 +42,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -199,7 +200,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (pRequest->body.queryFp) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -235,7 +236,8 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     setErrno(pRequest, code);
 
     if (pRequest->body.queryFp != NULL) {
-      pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+      doRequestCallback(pRequest, pRequest->code);
+
     } else {
       tsem_post(&pRequest->body.rspSem);
     }
@@ -299,7 +301,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -343,7 +345,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -380,7 +382,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -420,7 +422,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -534,13 +536,125 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
   return code;
 }
 
+static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  pBlock->info.hasVarCol = true;
+
+  pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData));
+
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  blockDataEnsureCapacity(pBlock, 1);
+
+  SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0);
+  SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1);
+  SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2);
+  char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0};
+  char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0};
+  if (pRsp->bAccepted) {
+    STR_TO_VARSTR(result, "accepted");
+    colDataSetVal(pResultCol, 0, result, false);
+    colDataSetVal(pIdCol, 0, (void*)&pRsp->compactId, false);
+    STR_TO_VARSTR(reason, "success");
+    colDataSetVal(pReasonCol, 0, reason, false);
+  } else {
+    STR_TO_VARSTR(result, "rejected");
+    colDataSetVal(pResultCol, 0, result, false);
+    colDataSetNULL(pIdCol, 0);
+    STR_TO_VARSTR(reason, "compaction is ongoing");
+    colDataSetVal(pReasonCol, 0, reason, false);
+  }
+  pBlock->info.rows = 1;
+
+  *block = pBlock;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetrieveTableRsp** pRsp) {
+  SSDataBlock* pBlock = NULL;
+  int32_t      code = buildCompactDbBlock(pCompactDb, &pBlock);
+  if (code) {
+    return code;
+  }
+
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
+  if (NULL == *pRsp) {
+    blockDataDestroy(pBlock);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  (*pRsp)->useconds = 0;
+  (*pRsp)->completed = 1;
+  (*pRsp)->precision = 0;
+  (*pRsp)->compressed = 0;
+  (*pRsp)->compLen = 0;
+  (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
+  (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
+
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
+  blockDataDestroy(pBlock);
+
+  if (len != rspSize - sizeof(SRetrieveTableRsp)) {
+    uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
+           (uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
+    return TSDB_CODE_TSC_INVALID_INPUT;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
+  SRequestObj* pRequest = param;
+  if (code != TSDB_CODE_SUCCESS) {
+    setErrno(pRequest, code);
+  } else {
+    SCompactDbRsp  rsp = {0};
+    SRetrieveTableRsp* pRes = NULL;
+    code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
+    }
+
+    if (code != 0) {
+      taosMemoryFree(pRes);
+    }
+  }
+
+  taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
+
+  if (pRequest->body.queryFp != NULL) {
+    pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
+  } else {
+    tsem_post(&pRequest->body.rspSem);
+  }
+  return code;  
+}
+
 __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
   switch (msgType) {
     case TDMT_MND_CONNECT:
@@ -557,6 +671,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
       return processAlterStbRsp;
     case TDMT_MND_SHOW_VARIABLES:
       return processShowVariablesRsp;
+    case TDMT_MND_COMPACT_DB:
+      return processCompactDbRsp;  
     default:
       return genericRspCallback;
   }

source/client/src/clientSml.c

@@ -20,75 +20,93 @@
 
 #include "clientSml.h"
 
-int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
-int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
-int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
+#define RETURN_FALSE                                 \
+  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
+  return false;
 
-//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
-//  NodeList *tmp = list;
-//  while (tmp) {
-//    if (fn == NULL) {
-//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return tmp->data.value;
-//      }
-//    } else {
-//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
-//        return tmp->data.value;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  return NULL;
-//}
-//
-//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
-//  NodeList *tmp = *list;
-//  while (tmp) {
-//    if (!tmp->data.used) break;
-//    if (fn == NULL) {
-//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return -1;
-//      }
-//    } else {
-//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
-//        return -1;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  if (tmp) {
-//    tmp->data.key = key;
-//    tmp->data.keyLen = len;
-//    tmp->data.value = value;
-//    tmp->data.used = true;
-//  } else {
-//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
-//    if (newNode == NULL) {
-//      return -1;
-//    }
-//    newNode->data.key = key;
-//    newNode->data.keyLen = len;
-//    newNode->data.value = value;
-//    newNode->data.used = true;
-//    newNode->next = *list;
-//    *list = newNode;
-//  }
-//  return 0;
-//}
-//
-//int nodeListSize(NodeList *list) {
-//  int cnt = 0;
-//  while (list) {
-//    if (list->data.used)
-//      cnt++;
-//    else
-//      break;
-//    list = list->next;
-//  }
-//  return cnt;
-//}
+#define SET_DOUBLE                     \
+  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
+  kvVal->d = result;
+
+#define SET_FLOAT                                                                              \
+  if (!IS_VALID_FLOAT(result)) {                                                               \
+    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
+    return false;                                                                              \
+  }                                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
+  kvVal->f = (float)result;
+
+#define SET_BIGINT                                                                                       \
+  errno = 0;                                                                                             \
+  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
+  if (errno == ERANGE) {                                                                                 \
+    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
+    return false;                                                                                        \
+  }                                                                                                      \
+  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
+  kvVal->i = tmp;
+
+#define SET_INT                                                                    \
+  if (!IS_VALID_INT(result)) {                                                     \
+    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
+    return false;                                                                  \
+  }                                                                                \
+  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
+  kvVal->i = result;
+
+#define SET_SMALL_INT                                                          \
+  if (!IS_VALID_SMALLINT(result)) {                                            \
+    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
+    return false;                                                              \
+  }                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
+  kvVal->i = result;
+
+#define SET_UBIGINT                                                                             \
+  errno = 0;                                                                                    \
+  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
+  if (errno == ERANGE || result < 0) {                                                          \
+    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
+    return false;                                                                               \
+  }                                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
+  kvVal->u = tmp;
+
+#define SET_UINT                                                                  \
+  if (!IS_VALID_UINT(result)) {                                                   \
+    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
+  kvVal->u = result;
+
+#define SET_USMALL_INT                                                            \
+  if (!IS_VALID_USMALLINT(result)) {                                              \
+    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
+  kvVal->u = result;
+
+#define SET_TINYINT                                                       \
+  if (!IS_VALID_TINYINT(result)) {                                        \
+    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
+    return false;                                                         \
+  }                                                                       \
+  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
+  kvVal->i = result;
+
+#define SET_UTINYINT                                                            \
+  if (!IS_VALID_UTINYINT(result)) {                                             \
+    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
+    return false;                                                               \
+  }                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
+  kvVal->u = result;
+
+int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
+int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
 
 static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
   SUserAuthInfo pAuth = {0};
@@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
-inline bool smlDoubleToInt64OverFlow(double num) {
-  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
-  return false;
-}
-
-void smlStrReplace(char* src, int32_t len){
-  if (!tsSmlDot2Underline) return;
-  for(int i = 0; i < len; i++){
-    if(src[i] == '.'){
-      src[i] = '_';
-    }
-  }
-}
 
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
@@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
   return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
 }
 
-int8_t smlGetTsTypeByLen(int32_t len) {
-  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
-    return TSDB_TIME_PRECISION_SECONDS;
-  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-    return TSDB_TIME_PRECISION_MILLI;
-  } else {
-    return -1;
-  }
-}
-
 SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
   SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
   if (!tag) {
@@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu
     goto cleanup;
   }
 
-  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
-  //  if (tag->tags == NULL) {
-  //    uError("SML:smlBuildTableInfo failed to allocate memory");
-  //    goto cleanup;
-  //  }
   return tag;
 
 cleanup:
@@ -192,6 +182,242 @@ cleanup:
   return NULL;
 }
 
+void smlBuildTsKv(SSmlKv *kv, int64_t ts){
+  kv->key = tsSmlTsDefaultName;
+  kv->keyLen = strlen(tsSmlTsDefaultName);
+  kv->type = TSDB_DATA_TYPE_TIMESTAMP;
+  kv->i = ts;
+  kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){
+  SSmlSTableMeta* sMeta = NULL;
+  char *measure = currElement->measure;
+  int   measureLen = currElement->measureLen;
+  if (currElement->measureEscaped) {
+    measure = (char *)taosMemoryMalloc(measureLen);
+    memcpy(measure, currElement->measure, measureLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+  }
+  STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+  if (currElement->measureEscaped) {
+    taosMemoryFree(measure);
+  }
+  if (pTableMeta == NULL) {
+    info->dataFormat = false;
+    info->reRun = true;
+    terrno = TSDB_CODE_SUCCESS;
+    return sMeta;
+  }
+  sMeta = smlBuildSTableMeta(info->dataFormat);
+  if(sMeta == NULL){
+    taosMemoryFreeClear(pTableMeta);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return sMeta;
+  }
+  sMeta->tableMeta = pTableMeta;
+  taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+  for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
+    SSchema *col = pTableMeta->schema + i;
+    SSmlKv   kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
+    if (col->type == TSDB_DATA_TYPE_NCHAR) {
+      kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) {
+      kv.length = col->bytes - VARSTR_HEADER_SIZE;
+    } else{
+      kv.length = col->bytes;
+    }
+
+    if(i < pTableMeta->tableInfo.numOfColumns){
+      taosArrayPush(sMeta->cols, &kv);
+    }else{
+      taosArrayPush(sMeta->tags, &kv);
+    }
+  }
+  return sMeta;
+}
+
+bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  // cnt begin 0, add ts so + 2
+  if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  // bind data
+  int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
+  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+    uDebug("smlBuildCol error, retry");
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  if (cnt >= taosArrayGetSize(info->maxColKVs)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+
+  if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+}
+
+bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+    goto END;
+  }
+
+  if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    goto END;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    goto END;
+  }
+
+  if (unlikely(kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+
+END:
+  info->dataFormat = false;
+  info->reRun = true;
+  return false;
+}
+
+int32_t smlJoinMeasureTag(SSmlLineInfo *elements){
+  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+  if(elements->measureTag == NULL){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
+  bool isSameMeasure = IS_SAME_SUPER_TABLE;
+  if(isSameMeasure) {
+    return 0;
+  }
+  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+
+  SSmlSTableMeta *sMeta = NULL;
+  if (unlikely(tmp == NULL)) {
+    sMeta = smlBuildSuperTableInfo(info, elements);
+    if(sMeta == NULL) return -1;
+  }else{
+    sMeta = *tmp;
+  }
+  ASSERT(sMeta != NULL);
+  info->currSTableMeta = sMeta->tableMeta;
+  info->maxTagKVs = sMeta->tags;
+  info->maxColKVs = sMeta->cols;
+  return 0;
+}
+
+int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
+  SSmlTableInfo **oneTable =
+      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
+  SSmlTableInfo *tinfo = NULL;
+  if (unlikely(oneTable == NULL)) {
+    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
+    if (unlikely(!tinfo)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES);
+
+    tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
+    for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
+      SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
+      if (kv->keyEscaped) kv->key = NULL;
+      if (kv->valueEscaped) kv->value = NULL;
+    }
+
+    smlSetCTableName(tinfo);
+    getTableUid(info, elements, tinfo);
+    if (info->dataFormat) {
+      info->currSTableMeta->uid = tinfo->uid;
+      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
+      if (tinfo->tableDataCtx == NULL) {
+        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
+        smlDestroyTableInfo(&tinfo);
+        return TSDB_CODE_SML_INVALID_DATA;
+      }
+    }
+  }else{
+    tinfo = *oneTable;
+  }
+  ASSERT(tinfo != NULL);
+  if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1);
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i);
+    if (elements->colArray == NULL) {
+      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
+    }
+    taosArrayPush(elements->colArray, kvTs);
+    taosArrayPush(elements->colArray, kv);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i);
+    taosArraySet(elements->colArray, 0, kvTs);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
   bool autoChildName = false;
   size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
@@ -328,98 +554,6 @@ cleanup:
   return NULL;
 }
 
-// uint16_t smlCalTypeSum(char* endptr, int32_t left){
-//   uint16_t sum = 0;
-//   for(int i = 0; i < left; i++){
-//     sum += endptr[i];
-//   }
-//   return sum;
-// }
-
-#define RETURN_FALSE                                 \
-  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-  return false;
-
-#define SET_DOUBLE                     \
-  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-  kvVal->d = result;
-
-#define SET_FLOAT                                                                              \
-  if (!IS_VALID_FLOAT(result)) {                                                               \
-    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-    return false;                                                                              \
-  }                                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-  kvVal->f = (float)result;
-
-#define SET_BIGINT                                                                                       \
-  errno = 0;                                                                                             \
-  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-  if (errno == ERANGE) {                                                                                 \
-    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-    return false;                                                                                        \
-  }                                                                                                      \
-  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-  kvVal->i = tmp;
-
-#define SET_INT                                                                    \
-  if (!IS_VALID_INT(result)) {                                                     \
-    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-    return false;                                                                  \
-  }                                                                                \
-  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-  kvVal->i = result;
-
-#define SET_SMALL_INT                                                          \
-  if (!IS_VALID_SMALLINT(result)) {                                            \
-    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-    return false;                                                              \
-  }                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-  kvVal->i = result;
-
-#define SET_UBIGINT                                                                             \
-  errno = 0;                                                                                    \
-  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-  if (errno == ERANGE || result < 0) {                                                          \
-    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-    return false;                                                                               \
-  }                                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-  kvVal->u = tmp;
-
-#define SET_UINT                                                                  \
-  if (!IS_VALID_UINT(result)) {                                                   \
-    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-  kvVal->u = result;
-
-#define SET_USMALL_INT                                                            \
-  if (!IS_VALID_USMALLINT(result)) {                                              \
-    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-  kvVal->u = result;
-
-#define SET_TINYINT                                                       \
-  if (!IS_VALID_TINYINT(result)) {                                        \
-    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
-    return false;                                                         \
-  }                                                                       \
-  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
-  kvVal->i = result;
-
-#define SET_UTINYINT                                                            \
-  if (!IS_VALID_UTINYINT(result)) {                                             \
-    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
-    return false;                                                               \
-  }                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
-  kvVal->u = result;
-
 bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
   const char *pVal = kvVal->value;
   int32_t     len = kvVal->length;
@@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
 static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                               ESchemaAction action) {
   SRequestObj   *pRequest = NULL;
@@ -1121,35 +1253,6 @@ end:
   return code;
 }
 
-/*
-static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
-  for(int i = 0; i < taosArrayGetSize(tags); i++) {
-    SSmlKv *tag = taosArrayGet(tags, i);
-    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
-      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
-      return TSDB_CODE_TSC_DUP_NAMES;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
-  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-
-  end:
-  taosHashCleanup(dumplicateKey);
-  return ret;
-}
-*/
-
 static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) {
     taosHashCleanup(kvHash);
   }
 
-  //  if (info->parseJsonByLib) {
-  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
-  //    if (key != NULL) taosMemoryFree(key->tags);
-  //  }
-  //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
@@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
 
-  // destroy info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // destroy info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
-
-  // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
   taosHashCleanup(info->childTables);
   taosHashCleanup(info->superTables);
@@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) {
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
-      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+      if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) {
+        taosMemoryFree(info->lines[i].measureTag);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
         return ret;
       }
     } else {
-      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
-      //      if (ret != TSDB_CODE_SUCCESS) {
-      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
-      //        return ret;
-      //      }
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
@@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
   uDebug(
       "SML:0x%" PRIx64
       " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
-        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
-      "",
+        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64,
       info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
       info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
       info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
       info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
       info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
+
 }
 
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
-  // clear info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(info, *oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // clear info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);

source/client/src/clientSmlJson.c

@@ -29,199 +29,6 @@
       (start)++;                 \
   }
 
-// SArray *smlJsonParseTags(char *start, char *end){
-//   SArray *tags = taosArrayInit(4, sizeof(SSmlKv));
-//   while(start < end){
-//     SSmlKv kv = {0};
-//     kv.type = TSDB_DATA_TYPE_NCHAR;
-//     bool isInQuote = false;
-//     while(start < end){
-//       if(unlikely(!isInQuote && *start == '"')){
-//         start++;
-//         kv.key = start;
-//         isInQuote = true;
-//         continue;
-//       }
-//       if(unlikely(isInQuote && *start == '"')){
-//         kv.keyLen = start - kv.key;
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//     bool hasColon = false;
-//     while(start < end){
-//       if(unlikely(!hasColon && *start == ':')){
-//         start++;
-//         hasColon = true;
-//         continue;
-//       }
-//       if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){
-//         kv.value = start;
-//         start++;
-//         continue;
-//       }
-//
-//       if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){
-//         kv.length = start - kv.value;
-//         taosArrayPush(tags, &kv);
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//   }
-//   return tags;
-// }
-
-// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) {
-//   int32_t ret = TSDB_CODE_SUCCESS;
-//
-//   if(is_same_child_table_telnet(elements, &info->preLine) == 0){
-//     return TSDB_CODE_SUCCESS;
-//   }
-//
-//   bool isSameMeasure = IS_SAME_SUPER_TABLE;
-//
-//   int     cnt = 0;
-//   SArray *preLineKV = info->preLineTagKV;
-//   bool    isSuperKVInit = true;
-//   SArray *superKV = NULL;
-//   if(info->dataFormat){
-//     if(unlikely(!isSameMeasure)){
-//       SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//       elements->measureLen, NULL);
-//
-//       if(unlikely(sMeta == NULL)){
-//         sMeta = smlBuildSTableMeta(info->dataFormat);
-//         STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-//         sMeta->tableMeta = pTableMeta;
-//         if(pTableMeta == NULL){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
-//       }
-//       info->currSTableMeta = sMeta->tableMeta;
-//       superKV = sMeta->tags;
-//
-//       if(unlikely(taosArrayGetSize(superKV) == 0)){
-//         isSuperKVInit = false;
-//       }
-//       taosArraySetSize(preLineKV, 0);
-//     }
-//   }else{
-//     taosArraySetSize(preLineKV, 0);
-//   }
-//
-//   SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen);
-//   int32_t tagNum = taosArrayGetSize(tags);
-//   if (tagNum == 0) {
-//     uError("SML:tag is empty:%s", elements->tags)
-//     taosArrayDestroy(tags);
-//     return TSDB_CODE_SML_INVALID_DATA;
-//   }
-//   for (int32_t i = 0; i < tagNum; ++i) {
-//     SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i);
-//
-//     if(info->dataFormat){
-//       if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){
-//         info->dataFormat = false;
-//         info->reRun      = true;
-//         taosArrayDestroy(tags);
-//         return TSDB_CODE_SUCCESS;
-//       }
-//
-//       if(isSameMeasure){
-//         if(unlikely(cnt >= taosArrayGetSize(preLineKV))) {
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-//         if(unlikely(kv.length > preKV->length)){
-//           preKV->length = kv.length;
-//           SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//           elements->measureLen, NULL);
-//            if(unlikely(NULL == tableMeta)){
-//              uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-//              return TSDB_CODE_SML_INTERNAL_ERROR;
-//            }
-//
-//           SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
-//           oldKV->length = kv.length;
-//           info->needModifySchema = true;
-//         }
-//         if(unlikely(!IS_SAME_KEY)){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//       }else{
-//         if(isSuperKVInit){
-//           if(unlikely(cnt >= taosArrayGetSize(superKV))) {
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//           SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-//           if(unlikely(kv.length > preKV->length)) {
-//             preKV->length = kv.length;
-//           }else{
-//             kv.length = preKV->length;
-//           }
-//           info->needModifySchema = true;
-//
-//           if(unlikely(!IS_SAME_KEY)){
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//         }else{
-//           taosArrayPush(superKV, &kv);
-//         }
-//         taosArrayPush(preLineKV, &kv);
-//       }
-//     }else{
-//       taosArrayPush(preLineKV, &kv);
-//     }
-//     cnt++;
-//   }
-//   taosArrayDestroy(tags);
-//
-//   SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES,
-//   is_same_child_table_telnet); if (unlikely(tinfo == NULL)) {
-//     tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-//     if (unlikely(!tinfo)) {
-//       return TSDB_CODE_OUT_OF_MEMORY;
-//     }
-//     tinfo->tags = taosArrayDup(preLineKV, NULL);
-//
-//     smlSetCTableName(tinfo);
-//     if (info->dataFormat) {
-//       info->currSTableMeta->uid = tinfo->uid;
-//       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-//       if (tinfo->tableDataCtx == NULL) {
-//         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-//         return TSDB_CODE_SML_INVALID_DATA;
-//       }
-//     }
-//
-//     SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-//     *key = *elements;
-//     tinfo->key = key;
-//     nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
-//   }
-//   if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
-//
-//   return ret;
-// }
-
 static char *smlJsonGetObj(char *payload) {
   int  leftBracketCnt = 0;
   bool isInQuote = false;
@@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
       break;
     }
     case cJSON_String: {
-      /* set default JSON type to binary/nchar according to
-       * user configured parameter tsDefaultJSONStrType
-       */
-
-      char *tsDefaultJSONStrType = "binary";  // todo
-      smlConvertJSONString(kv, tsDefaultJSONStrType, root);
+      smlConvertJSONString(kv, "binary", root);
       break;
     }
     case cJSON_Object: {
@@ -682,138 +484,70 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
-  int32_t ret = TSDB_CODE_SUCCESS;
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   int32_t tagNum = cJSON_GetArraySize(tags);
   if (unlikely(tagNum == 0)) {
     uError("SML:Tag should not be empty");
-    return TSDB_CODE_TSC_INVALID_JSON;
+    terrno = TSDB_CODE_TSC_INVALID_JSON;
+    return -1;
   }
   for (int32_t i = 0; i < tagNum; ++i) {
     cJSON *tag = cJSON_GetArrayItem(tags, i);
     if (unlikely(tag == NULL)) {
-      return TSDB_CODE_TSC_INVALID_JSON;
+      terrno = TSDB_CODE_TSC_INVALID_JSON;
+      return -1;
     }
-    //    if(unlikely(tag == cMeasure)) continue;
     size_t keyLen = strlen(tag->string);
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
       uError("OTD:Tag key length is 0 or too large than 64");
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno =  TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // add kv to SSmlKv
-    SSmlKv kv = {.key = tag->string, .keyLen = keyLen};
+    SSmlKv kv = {0};
+    kv.key = tag->string;
+    kv.keyLen = keyLen;
+
     // value
-    ret = smlParseValueFromJSON(tag, &kv);
+    int32_t ret = smlParseValueFromJSON(tag, &kv);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      return ret;
-    }
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      terrno =  ret;
+      return -1;
     }
     taosArrayPush(preLineKV, &kv);
+
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno =  TSDB_CODE_SUCCESS;
+      return -1;
+    }
+
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (unlikely(!tinfo)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    if(info->parseJsonByLib){
-    //      key->tags = taosMemoryMalloc(elements->tagsLen + 1);
-    //      memcpy(key->tags, elements->tags, elements->tagsLen);
-    //      key->tags[elements->tagsLen] = 0;
-    //    }
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
-  return ret;
+  ret = smlProcessTagJson(info, tags);
+  if(ret != 0){
+    return terrno;
+  }
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+  return smlProcessChildTable(info, elements);
 }
 
 static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
@@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
@@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-
   if (unlikely(info->lines != NULL)) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
@@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 int32_t smlParseJSON(SSmlHandle *info, char *payload) {

source/client/src/clientSmlLine.c

@@ -20,15 +20,9 @@
 
 #include "clientSml.h"
 
-// comma ,
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-// space
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-// equal =
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
-// quote "
-// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-// SLASH
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
@@ -51,10 +45,10 @@
     }                                                \
   }
 
-#define BINARY_ADD_LEN 2  // "binary"   2 means " "
-#define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
+#define BINARY_ADD_LEN (sizeof("\"\"")-1)    // "binary"   2 means length of ("")
+#define NCHAR_ADD_LEN  (sizeof("L\"\"")-1)   // L"nchar"   3 means length of (L"")
 
-uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
+uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
                                   TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO,
                                   TSDB_TIME_PRECISION_NANO};
 
@@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   return TSDB_CODE_TSC_INVALID_VALUE;
 }
 
-static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
-  if (isSameCTable) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int     cnt = 0;
+static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
-        for (int i = pTableMeta->tableInfo.numOfColumns;
-             i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name,
-                         .keyLen = strlen(tag->name),
-                         .type = tag->type,
-                         .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE};
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
   taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
@@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     while (*sql < sqlEnd) {
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(IS_EQUAL(*sql))) {
         keyLen = *sql - key;
@@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // parse value
@@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         break;
       } else if (unlikely(IS_EQUAL(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(valueLen == 0)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_SML_INVALID_DATA;
+      terrno = TSDB_CODE_SML_INVALID_DATA;
+      return -1;
     }
 
     if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
     if (keyEscaped) {
@@ -331,37 +279,17 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
       value = tmp;
     }
     SSmlKv kv = {.key = key,
-                 .keyLen = keyLen,
-                 .type = TSDB_DATA_TYPE_NCHAR,
-                 .value = value,
-                 .length = valueLen,
-                 .keyEscaped = keyEscaped,
-                 .valueEscaped = valueEscaped};
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = keyEscaped,
+        .valueEscaped = valueEscaped};
     taosArrayPush(preLineKV, &kv);
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
 
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
     }
 
     cnt++;
@@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
     (*sql)++;
   }
+  return 0;
+}
 
-  void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-  if ((oneTable != NULL)) {
+static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
+  bool isSameCTable = IS_SAME_CHILD_TABLE;
+  if(isSameCTable){
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen);
-  if (unlikely(!tinfo)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tinfo->tags = taosArrayDup(preLineKV, NULL);
-  for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
-    if (kv->keyEscaped) kv->key = NULL;
-    if (kv->valueEscaped) kv->value = NULL;
-  }
-
-  smlSetCTableName(tinfo);
-  getTableUid(info, currElement, tinfo);
-  if (info->dataFormat) {
-    info->currSTableMeta->uid = tinfo->uid;
-    tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-    if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(&tinfo);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-      return TSDB_CODE_SML_INVALID_DATA;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
   }
 
-  taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
+  ret = smlProcessTagLine(info, sql, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
 
-  return TSDB_CODE_SUCCESS;
+  return smlProcessChildTable(info, elements);
 }
 
-static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
+static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
   int cnt = 0;
-  if (info->dataFormat) {
-    if (unlikely(!isSameCTable)) {
-      SSmlTableInfo **oneTable =
-          (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-      if (unlikely(oneTable == NULL)) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-      info->currTableDataCtx = (*oneTable)->tableDataCtx;
-    }
-
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        *tmp = smlBuildSTableMeta(info->dataFormat);
-        if(*tmp == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
-
-        for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
-          if (tag->type == TSDB_DATA_TYPE_NCHAR) {
-            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
-          } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
-            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
-          }
-          taosArrayPush((*tmp)->cols, &kv);
-        }
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxColKVs = (*tmp)->cols;
-    }
-  }
-
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
@@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
 
     if (info->dataFormat) {
-      // cnt begin 0, add ts so + 2
-      if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      // bind data
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1);
-      if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-        uDebug("smlBuildCol error, retry");
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (cnt >= taosArrayGetSize(info->maxColKVs)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
-      if (kv.type != maxKV->type) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      bool isAligned = isSmlColAligned(info, cnt, &kv);
       freeSSmlKv(&kv);
+      if(!isAligned){
+        return TSDB_CODE_SUCCESS;
+      }
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA;
   elements->measure = sql;
-
   // parse measure
   size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
@@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
-
-  bool isSameCTable = false;
-  bool isSameMeasure = false;
-  if (IS_SAME_CHILD_TABLE) {
-    isSameCTable = true;
-    isSameMeasure = true;
-  } else if (info->dataFormat) {
-    isSameMeasure = IS_SAME_SUPER_TABLE;
-  }
+  elements->measureTag = elements->measure;
   // parse tag
   if (*sql == COMMA) sql++;
   elements->tags = sql;
 
-  int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  int ret = smlParseTagLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   elements->cols = sql;
 
-  ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  ret = smlParseColLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  // add ts to
-  SSmlKv kv = {.key = tsSmlTsDefaultName,
-               .keyLen = strlen(tsSmlTsDefaultName),
-               .type = TSDB_DATA_TYPE_TIMESTAMP,
-               .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
-               .keyEscaped = false,
-               .valueEscaped = false};
-  if (info->dataFormat) {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
 
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
-    taosArraySet(elements->colArray, 0, &kv);
-  }
-  info->preLine = *elements;
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  return ret;
+  return smlParseEndLine(info, elements, &kvTs);
 }

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,6 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b) {
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  //  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-  //         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
     return 1;
   return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
@@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
   }
 }
 
-static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
-  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-    elements->measureTag = info->preLine.measureTag;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (!isSameMeasure) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
-  taosArrayClear(preLineKV);
   const char *sql = data;
   while (sql < sqlEnd) {
     JUMP_SPACE(sql, sqlEnd)
@@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     // parse key
     while (sql < sqlEnd) {
       if (unlikely(*sql == SPACE)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(*sql == EQUAL)) {
         keyLen = sql - key;
@@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     }
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
-      smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
-    //    if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) {
-    //      smlBuildInvalidDataMsg(msg, "dumplicate key", key);
-    //      return TSDB_CODE_TSC_DUP_NAMES;
-    //    }
 
     // parse value
     const char *value = sql;
@@ -150,87 +110,67 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         break;
       }
       if (unlikely(*sql == EQUAL)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       sql++;
     }
     valueLen = sql - value;
 
     if (unlikely(valueLen == 0)) {
-      smlBuildInvalidDataMsg(msg, "invalid value", value);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
+      terrno = TSDB_CODE_TSC_INVALID_VALUE;
+      return -1;
     }
 
     if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-    }
+    SSmlKv kv = {.key = key,
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = false,
+        .valueEscaped = false};
     taosArrayPush(preLineKV, &kv);
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
+    }
     cnt++;
   }
+  return 0;
+}
 
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (!tinfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-    }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
+static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
+  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
+    elements->measureTag = info->preLine.measureTag;
+    return TSDB_CODE_SUCCESS;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
 
-  return TSDB_CODE_SUCCESS;
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
+    }
+  }
+
+  ret = smlProcessTagTelnet(info, data, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+
+  return smlProcessChildTable(info, elements);
 }
 
 // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
@@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
-  bool needConverTime = false;  // get TS before parse tag(get meta), so need conver time
+  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
   if (info->dataFormat && info->currSTableMeta == NULL) {
     needConverTime = true;
   }
@@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
   // parse value
   smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
@@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf);
+  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SUCCESS;
   }
 
-  if (info->dataFormat && info->currSTableMeta != NULL) {
-    if (needConverTime) {
-      kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
-    }
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
+  if (needConverTime) {
+    kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
-  info->preLine = *elements;
 
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }

source/common/CMakeLists.txt

@@ -60,8 +60,8 @@ if(${BUILD_S3})
         find_library(S3_LIBRARY s3)
         find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
         find_library(XML2_LIBRARY xml2)
-        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
+        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
         target_link_libraries(
                 common
 

source/common/src/cos.c

@@ -1,6 +1,8 @@
 #define ALLOW_FORBID_FUNC
 
 #include "cos.h"
+#include "cos_cp.h"
+#include "tdef.h"
 
 extern char   tsS3Endpoint[];
 extern char   tsS3AccessKeyId[];
@@ -66,11 +68,12 @@ static int should_retry() {
   return 0;
 }
 
-static void s3PrintError(const char *func, S3Status status, char error_details[]) {
+static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
+                         char error_details[]) {
   if (status < S3StatusErrorAccessDenied) {
-    uError("%s: %s", __func__, S3_get_status_name(status));
+    uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status));
   } else {
-    uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+    uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details);
   }
 }
 
@@ -230,9 +233,9 @@ typedef struct put_object_callback_data {
   int         noStatus;
 } put_object_callback_data;
 
-#define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
+#define MULTIPART_CHUNK_SIZE (64 << 20)  // multipart is 768M
 
-typedef struct UploadManager {
+typedef struct {
   char     err_msg[512];
   S3Status status;
   uint64_t content_length;
@@ -307,6 +310,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat
 S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   manager->upload_id = strdup(upload_id);
+  manager->status = S3StatusOK;
   return S3StatusOK;
 }
 
@@ -321,6 +325,17 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti
   return S3StatusOK;
 }
 
+S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) {
+  responsePropertiesCallbackNull(properties, callbackData);
+
+  MultipartPartData *data = (MultipartPartData *)callbackData;
+  int                seq = data->seq;
+  const char        *etag = properties->eTag;
+  data->manager->etags[seq - 1] = strdup(etag);
+  // data->manager->next_etags_pos = seq;
+  return S3StatusOK;
+}
+
 static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   int            ret = 0;
@@ -437,17 +452,330 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
       // printListMultipartHeader(data.allDetails);
     }
   } else {
-    s3PrintError(__func__, data.status, data.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
     return -1;
   }
 
   return 0;
 }
 */
-int32_t s3PutObjectFromFile2(const char *file, const char *object) {
-  int32_t     code = 0;
-  const char *key = object;
-  // const char              *uploadId = 0;
+
+static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char const *object_name, int64_t size,
+                                         S3PutProperties *put_prop, put_object_callback_data *data) {
+  int32_t            code = 0;
+  S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  do {
+    S3_put_object(bucket_context, object_name, size, put_prop, 0, 0, &putObjectHandler, data);
+  } while (S3_status_is_retryable(data->status) && should_retry());
+
+  if (data->status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+  } else if (data->contentLength) {
+    uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
+    code = TAOS_SYSTEM_ERROR(EIO);
+  }
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
+                                            int64_t contentLength, S3PutProperties *put_prop,
+                                            put_object_callback_data *data) {
+  int32_t       code = 0;
+  uint64_t      totalContentLength = contentLength;
+  uint64_t      todoContentLength = contentLength;
+  UploadManager manager = {0};
+
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
+  int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  const int max_part_num = 10000;
+  if (totalSeq > max_part_num) {
+    chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
+    totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  }
+
+  MultipartPartData partData;
+  memset(&partData, 0, sizeof(MultipartPartData));
+  int partContentLength = 0;
+
+  S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                       &initial_multipart_callback};
+
+  S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  S3MultipartCommitHandler commit_handler = {
+      {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
+  manager.next_etags_pos = 0;
+  do {
+    S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+
+  if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+upload:
+  todoContentLength -= chunk_size * manager.next_etags_pos;
+  for (int seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
+    partData.manager = &manager;
+    partData.seq = seq;
+    if (partData.put_object_data.gb == NULL) {
+      partData.put_object_data = *data;
+    }
+    partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
+    // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
+    partData.put_object_data.contentLength = partContentLength;
+    partData.put_object_data.originalContentLength = partContentLength;
+    partData.put_object_data.totalContentLength = todoContentLength;
+    partData.put_object_data.totalOriginalContentLength = totalContentLength;
+    put_prop->md5 = 0;
+    do {
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
+                     partContentLength, 0, timeoutMsG, &partData);
+    } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
+    if (partData.put_object_data.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+    contentLength -= chunk_size;
+    todoContentLength -= chunk_size;
+  }
+
+  int i;
+  int size = 0;
+  size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+  char buf[256];
+  int  n;
+  for (i = 0; i < totalSeq; i++) {
+    if (!manager.etags[i]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+    n = snprintf(buf, sizeof(buf),
+                 "<Part><PartNumber>%d</PartNumber>"
+                 "<ETag>%s</ETag></Part>",
+                 i + 1, manager.etags[i]);
+    size += growbuffer_append(&(manager.gb), buf, n);
+  }
+  size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+  manager.remaining = size;
+
+  do {
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                 timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+  if (manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+clean:
+  if (manager.upload_id) {
+    taosMemoryFree(manager.upload_id);
+  }
+  for (i = 0; i < manager.next_etags_pos; i++) {
+    taosMemoryFree(manager.etags[i]);
+  }
+  growbuffer_destroy(manager.gb);
+  taosMemoryFree(manager.etags);
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
+                                         char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
+                                         put_object_callback_data *data) {
+  int32_t code = 0;
+
+  uint64_t totalContentLength = contentLength;
+  // uint64_t      todoContentLength = contentLength;
+  UploadManager manager = {0};
+
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
+  int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  const int max_part_num = 10000;
+  if (totalSeq > max_part_num) {
+    chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
+    totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  }
+
+  bool need_init_upload = true;
+  char file_cp_path[TSDB_FILENAME_LEN];
+  snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
+
+  SCheckpoint cp = {0};
+  cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
+
+  if (taosCheckExistFile(file_cp_path)) {
+    if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
+      manager.upload_id = strdup(cp.upload_id);
+      need_init_upload = false;
+    } else {
+      cos_cp_remove(file_cp_path);
+    }
+  }
+
+  if (need_init_upload) {
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &initial_multipart_callback};
+    do {
+      S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
+    } while (S3_status_is_retryable(manager.status) && should_retry());
+
+    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
+  }
+
+  if (cos_cp_open(file_cp_path, &cp)) {
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  int     part_num = 0;
+  int64_t consume_bytes = 0;
+  // SCheckpointPart *parts = taosMemoryCalloc(cp.part_num, sizeof(SCheckpointPart));
+  // cos_cp_get_undo_parts(&cp, &part_num, parts, &consume_bytes);
+
+  MultipartPartData partData;
+  memset(&partData, 0, sizeof(MultipartPartData));
+  int partContentLength = 0;
+
+  S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallbackWithCp, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  S3MultipartCommitHandler commit_handler = {
+      {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
+  manager.next_etags_pos = 0;
+
+upload:
+  // todoContentLength -= chunk_size * manager.next_etags_pos;
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (cp.parts[i].completed) {
+      continue;
+    }
+
+    if (i > 0 && cp.parts[i - 1].completed) {
+      if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
+        code = TAOS_SYSTEM_ERROR(errno);
+        goto clean;
+      }
+    }
+
+    int seq = cp.parts[i].index + 1;
+
+    partData.manager = &manager;
+    partData.seq = seq;
+    if (partData.put_object_data.gb == NULL) {
+      partData.put_object_data = *data;
+    }
+
+    partContentLength = cp.parts[i].size;
+    partData.put_object_data.contentLength = partContentLength;
+    partData.put_object_data.originalContentLength = partContentLength;
+    // partData.put_object_data.totalContentLength = todoContentLength;
+    partData.put_object_data.totalOriginalContentLength = totalContentLength;
+    put_prop->md5 = 0;
+    do {
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
+                     partContentLength, 0, timeoutMsG, &partData);
+    } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
+    if (partData.put_object_data.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+
+      //(void)cos_cp_dump(&cp);
+      goto clean;
+    }
+
+    if (!manager.etags[seq - 1]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
+    (void)cos_cp_dump(&cp);
+
+    contentLength -= chunk_size;
+    // todoContentLength -= chunk_size;
+  }
+
+  cos_cp_close(cp.thefile);
+  cp.thefile = 0;
+
+  int size = 0;
+  size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+  char buf[256];
+  int  n;
+  for (int i = 0; i < cp.part_num; ++i) {
+    n = snprintf(buf, sizeof(buf),
+                 "<Part><PartNumber>%d</PartNumber>"
+                 "<ETag>%s</ETag></Part>",
+                 // i + 1, manager.etags[i]);
+                 cp.parts[i].index + 1, cp.parts[i].etag);
+    size += growbuffer_append(&(manager.gb), buf, n);
+  }
+  size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+  manager.remaining = size;
+
+  do {
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                 timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+  if (manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  cos_cp_remove(file_cp_path);
+
+clean:
+  /*
+  if (parts) {
+    taosMemoryFree(parts);
+  }
+  */
+  if (cp.thefile) {
+    cos_cp_close(cp.thefile);
+  }
+  if (cp.parts) {
+    taosMemoryFree(cp.parts);
+  }
+
+  if (manager.upload_id) {
+    taosMemoryFree(manager.upload_id);
+  }
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (manager.etags[i]) {
+      taosMemoryFree(manager.etags[i]);
+    }
+  }
+  taosMemoryFree(manager.etags);
+  growbuffer_destroy(manager.gb);
+
+  return code;
+}
+
+int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
+  int32_t                  code = 0;
+  int32_t                  lmtime = 0;
   const char              *filename = 0;
   uint64_t                 contentLength = 0;
   const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
@@ -458,22 +786,16 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   S3NameValue              metaProperties[S3_MAX_METADATA_COUNT];
   char                     useServerSideEncryption = 0;
   put_object_callback_data data = {0};
-  // int                      noStatus = 0;
 
-  // data.infile = 0;
-  // data.gb = 0;
-  // data.infileFD = NULL;
-  // data.noStatus = noStatus;
-
-  if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
-    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
+  if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
     return code;
   }
 
   if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
-    uError("ERROR: %s Failed to open file %s: ", __func__, file);
     code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open file %s: ", __func__, file);
     return code;
   }
 
@@ -490,145 +812,19 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
                                    metaProperties,  useServerSideEncryption};
 
   if (contentLength <= MULTIPART_CHUNK_SIZE) {
-    S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
-                                           &putObjectDataCallback};
-
-    do {
-      S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
-    } while (S3_status_is_retryable(data.status) && should_retry());
-
-    if (data.status != S3StatusOK) {
-      s3PrintError(__func__, data.status, data.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-    } else if (data.contentLength) {
-      uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
-             (unsigned long long)data.contentLength);
-      code = TAOS_SYSTEM_ERROR(EIO);
-    }
+    code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
   } else {
-    uint64_t      totalContentLength = contentLength;
-    uint64_t      todoContentLength = contentLength;
-    UploadManager manager;
-    manager.upload_id = 0;
-    manager.gb = 0;
-
-    // div round up
-    int       seq;
-    uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 7;
-    int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
-    const int max_part_num = 1000;
-    if (totalSeq > max_part_num) {
-      chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
-      totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+    if (withcp) {
+      code = s3PutObjectFromFileWithCp(&bucketContext, file, lmtime, object_name, contentLength, &putProperties, &data);
+    } else {
+      code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
     }
+  }
 
-    MultipartPartData partData;
-    memset(&partData, 0, sizeof(MultipartPartData));
-    int partContentLength = 0;
-
-    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
-                                         &initial_multipart_callback};
-
-    S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback},
-                                           &putObjectDataCallback};
-
-    S3MultipartCommitHandler commit_handler = {
-        {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
-
-    manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
-    manager.next_etags_pos = 0;
-    /*
-    if (uploadId) {
-      manager.upload_id = strdup(uploadId);
-      manager.remaining = contentLength;
-      if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
-        fseek(data.infile, -(manager.remaining), 2);
-        taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
-        contentLength = manager.remaining;
-        goto upload;
-      } else {
-        goto clean;
-      }
-    }
-    */
-    do {
-      S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
-    } while (S3_status_is_retryable(manager.status) && should_retry());
-
-    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
-      s3PrintError(__func__, manager.status, manager.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-      goto clean;
-    }
-
-  upload:
-    todoContentLength -= chunk_size * manager.next_etags_pos;
-    for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
-      partData.manager = &manager;
-      partData.seq = seq;
-      if (partData.put_object_data.gb == NULL) {
-        partData.put_object_data = data;
-      }
-      partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
-      // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
-      partData.put_object_data.contentLength = partContentLength;
-      partData.put_object_data.originalContentLength = partContentLength;
-      partData.put_object_data.totalContentLength = todoContentLength;
-      partData.put_object_data.totalOriginalContentLength = totalContentLength;
-      putProperties.md5 = 0;
-      do {
-        S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
-                       partContentLength, 0, timeoutMsG, &partData);
-      } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
-      if (partData.put_object_data.status != S3StatusOK) {
-        s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg);
-        code = TAOS_SYSTEM_ERROR(EIO);
-        goto clean;
-      }
-      contentLength -= chunk_size;
-      todoContentLength -= chunk_size;
-    }
-
-    int i;
-    int size = 0;
-    size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
-    char buf[256];
-    int  n;
-    for (i = 0; i < totalSeq; i++) {
-      n = snprintf(buf, sizeof(buf),
-                   "<Part><PartNumber>%d</PartNumber>"
-                   "<ETag>%s</ETag></Part>",
-                   i + 1, manager.etags[i]);
-      size += growbuffer_append(&(manager.gb), buf, n);
-    }
-    size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
-    manager.remaining = size;
-
-    do {
-      S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
-                                   timeoutMsG, &manager);
-    } while (S3_status_is_retryable(manager.status) && should_retry());
-    if (manager.status != S3StatusOK) {
-      s3PrintError(__func__, manager.status, manager.err_msg);
-      code = TAOS_SYSTEM_ERROR(EIO);
-      goto clean;
-    }
-
-  clean:
-    if (manager.upload_id) {
-      taosMemoryFree(manager.upload_id);
-    }
-    for (i = 0; i < manager.next_etags_pos; i++) {
-      taosMemoryFree(manager.etags[i]);
-    }
-    if (data.infileFD) {
-      taosCloseFile(&data.infileFD);
-    } else if (data.gb) {
-      growbuffer_destroy(data.gb);
-    }
-
-    growbuffer_destroy(manager.gb);
-    taosMemoryFree(manager.etags);
+  if (data.infileFD) {
+    taosCloseFile(&data.infileFD);
+  } else if (data.gb) {
+    growbuffer_destroy(data.gb);
   }
 
   return code;
@@ -722,7 +918,7 @@ static SArray *getListByPrefix(const char *prefix) {
       return data.objectArray;
     }
   } else {
-    s3PrintError(__func__, data.status, data.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
   }
 
   taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
@@ -741,7 +937,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
     } while (S3_status_is_retryable(cbd.status) && should_retry());
 
     if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-      s3PrintError(__func__, cbd.status, cbd.err_msg);
+      s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
     }
   }
 }
@@ -884,7 +1080,9 @@ long s3Size(const char *object_name) {
   } while (S3_status_is_retryable(cbd.status) && should_retry());
 
   if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
+
+    return -1;
   }
 
   size = cbd.content_length;
@@ -974,7 +1172,7 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
   return code;
 }
 
-int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
+int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
   int32_t                     code = 0;
   cos_pool_t                 *p = NULL;
   int                         is_cname = 0;
@@ -986,6 +1184,7 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
   cos_table_t                *headers = NULL;
   cos_resumable_clt_params_t *clt_params = NULL;
 
+  (void)withcp;
   cos_pool_create(&p, NULL);
   options = cos_request_options_create(p);
   s3InitRequestOptions(options, is_cname);
@@ -1335,7 +1534,7 @@ long s3Size(const char *object_name) {
 int32_t s3Init() { return 0; }
 void    s3CleanUp() {}
 int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
-int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
 void    s3DeleteObjectsByPrefix(const char *prefix) {}
 void    s3DeleteObjects(const char *object_name[], int nobject) {}
 bool    s3Exists(const char *object_name) { return false; }
@@ -1346,5 +1545,6 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
 void    s3EvictCache(const char *path, long object_size) {}
 long    s3Size(const char *object_name) { return 0; }
 int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; }
+int32_t s3GetObjectToFile(const char *object_name, char *fileName) { return 0; }
 
 #endif

source/common/src/cos_cp.c

@@ -0,0 +1,417 @@
+#define ALLOW_FORBID_FUNC
+
+#include "cos_cp.h"
+#include "cJSON.h"
+#include "tutil.h"
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, cp_path);
+    return code;
+  }
+
+  checkpoint->thefile = fd;
+
+  return code;
+}
+
+void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
+void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
+
+static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
+  int32_t      code = 0;
+  cJSON const* item2 = NULL;
+
+  cJSON* json = cJSON_Parse(cp_body);
+  if (NULL == json) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+
+  cJSON const* item = cJSON_GetObjectItem(json, "ver");
+  if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
+    goto _exit;
+  }
+
+  item = cJSON_GetObjectItem(json, "type");
+  if (!cJSON_IsNumber(item)) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+  cp->cp_type = item->valuedouble;
+
+  item = cJSON_GetObjectItem(json, "md5");
+  if (cJSON_IsString(item)) {
+    memcpy(cp->md5, item->valuestring, strlen(item->valuestring));
+  }
+
+  item = cJSON_GetObjectItem(json, "upload_id");
+  if (cJSON_IsString(item)) {
+    strncpy(cp->upload_id, item->valuestring, 128);
+  }
+
+  item2 = cJSON_GetObjectItem(json, "file");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->file_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "lastmodified");
+    if (cJSON_IsNumber(item)) {
+      cp->file_last_modified = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "path");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_path, item->valuestring, TSDB_FILENAME_LEN);
+    }
+
+    item = cJSON_GetObjectItem(item2, "file_md5");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_md5, item->valuestring, 64);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "object");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "object_size");
+    if (cJSON_IsNumber(item)) {
+      cp->object_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_name");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_name, item->valuestring, 128);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_last_modified");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_last_modified, item->valuestring, 64);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_etag");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_etag, item->valuestring, 128);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "cpparts");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "number");
+    if (cJSON_IsNumber(item)) {
+      cp->part_num = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->part_size = item->valuedouble;
+    }
+
+    item2 = cJSON_GetObjectItem(item2, "parts");
+    if (cJSON_IsArray(item2) && cp->part_num > 0) {
+      cJSON_ArrayForEach(item, item2) {
+        cJSON const* item3 = cJSON_GetObjectItem(item, "index");
+        int32_t      index = 0;
+        if (cJSON_IsNumber(item3)) {
+          index = item3->valuedouble;
+          cp->parts[index].index = index;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "offset");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].offset = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "size");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].size = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "completed");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].completed = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "crc64");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].crc64 = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "etag");
+        if (cJSON_IsString(item3)) {
+          strncpy(cp->parts[index].etag, item3->valuestring, 128);
+        }
+      }
+    }
+  }
+
+_exit:
+  if (json) cJSON_Delete(json);
+  if (cp_body) taosMemoryFree(cp_body);
+
+  return code;
+}
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, filepath);
+    goto _exit;
+  }
+
+  int64_t size = -1;
+  code = taosStatFile(filepath, &size, NULL, NULL);
+  if (code) {
+    uError("ERROR: %s Failed to stat %s", __func__, filepath);
+    goto _exit;
+  }
+
+  char* cp_body = taosMemoryMalloc(size + 1);
+
+  int64_t n = taosReadFile(fd, cp_body, size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to read %s", __func__, filepath);
+    goto _exit;
+  } else if (n != size) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
+    goto _exit;
+  }
+  taosCloseFile(&fd);
+  cp_body[size] = '\0';
+
+  return cos_cp_parse_body(cp_body, checkpoint);
+
+_exit:
+  if (fd) {
+    taosCloseFile(&fd);
+  }
+  if (cp_body) {
+    taosMemoryFree(cp_body);
+  }
+
+  return code;
+}
+
+static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  char* data = cJSON_PrintUnformatted(json);
+  if (NULL == data) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  TdFilePtr fp = checkpoint->thefile;
+  if (taosFtruncateFile(fp, 0) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosWriteFile(fp, data, strlen(data)) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+  if (taosFsyncFile(fp) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+_exit:
+  taosMemoryFree(data);
+  return code;
+}
+
+int32_t cos_cp_dump(SCheckpoint* cp) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  cJSON* ojson = NULL;
+  cJSON* json = cJSON_CreateObject();
+  if (!json) return TSDB_CODE_OUT_OF_MEMORY;
+
+  if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "file");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "object");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  ojson = cJSON_AddObjectToObject(json, "cpparts");
+  if (!ojson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
+  if (!ajson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  for (int i = 0; i < cp->part_num; ++i) {
+    cJSON* item = cJSON_CreateObject();
+    if (!item) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    cJSON_AddItemToArray(ajson, item);
+
+    if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  code = cos_cp_save_json(json, cp);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+  }
+  cJSON_Delete(json);
+  return code;
+}
+
+void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}
+
+void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
+  checkpoint->parts[part_index].completed = 1;
+  strncpy(checkpoint->parts[part_index].etag, etag, 128);
+  checkpoint->parts[part_index].crc64 = crc64;
+}
+
+void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                         char const* upload_id, int64_t part_size) {
+  int i = 0;
+
+  checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
+  strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
+
+  checkpoint->file_size = size;
+  checkpoint->file_last_modified = mtime;
+  strncpy(checkpoint->upload_id, upload_id, 128);
+
+  checkpoint->part_size = part_size;
+  for (; i * part_size < size; i++) {
+    checkpoint->parts[i].index = i;
+    checkpoint->parts[i].offset = i * part_size;
+    checkpoint->parts[i].size = TMIN(part_size, (size - i * part_size));
+    checkpoint->parts[i].completed = 0;
+    checkpoint->parts[i].etag[0] = '\0';
+  }
+  checkpoint->part_num = i;
+}
+
+static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
+
+bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
+  if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
+    return true;
+  }
+
+  return false;
+}

source/common/src/systable.c

@@ -140,7 +140,7 @@ static const SSysDbTableSchema userStbsSchema[] = {
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
-    {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "watermark", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "max_delay", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "rollup", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@@ -167,7 +167,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "level", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "status", .bytes = 15 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "stage", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@@ -334,6 +334,20 @@ static const SSysDbTableSchema userViewsSchema[] = {
 //    {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 };
 
+static const SSysDbTableSchema userCompactsSchema[] = {
+    {.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+};
+
+static const SSysDbTableSchema userCompactsDetailSchema[] = {
+    {.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+};
 
 static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
@@ -362,6 +376,8 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
     {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
     {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
+    {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
+    {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tdataformat.c

@@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
     n += tPutCStr(p ? p + n : p, pTagVal->pKey);
   } else {
     n += tPutI16v(p ? p + n : p, pTagVal->cid);
+    ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
   }
 
   // type
@@ -3590,9 +3591,9 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_
     tColDataCalcSMAUInt,           // TSDB_DATA_TYPE_UINT
     tColDataCalcSMAUBigInt,        // TSDB_DATA_TYPE_UBIGINT
     tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_JSON
-    NULL,                          // TSDB_DATA_TYPE_VARBINARY
-    NULL,                          // TSDB_DATA_TYPE_DECIMAL
-    NULL,                          // TSDB_DATA_TYPE_BLOB
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_VARBINARY
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_DECIMAL
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_BLOB
     NULL,                          // TSDB_DATA_TYPE_MEDIUMBLOB
     tColDataCalcSMAVarType         // TSDB_DATA_TYPE_GEOMETRY
 };

source/common/src/tglobal.c

@@ -108,7 +108,7 @@ bool    tsEnableTelem = true;
 int32_t  tsTelemInterval = 43200;
 char     tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com";
 uint16_t tsTelemPort = 80;
-char    *tsTelemUri = "/report";
+char *   tsTelemUri = "/report";
 
 #ifdef TD_ENTERPRISE
 bool tsEnableCrashReport = false;
@@ -248,6 +248,7 @@ int32_t tsTtlBatchDropNum = 10000;   // number of tables dropped per batch
 
 // internal
 int32_t tsTransPullupInterval = 2;
+int32_t tsCompactPullupInterval = 10;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointInterval = 60;
 float   tsSinkDataRate = 2.0;
@@ -279,7 +280,9 @@ char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
 int32_t tsS3BlockSize = -1;        // number of tsdb pages (4096)
 int32_t tsS3BlockCacheSize = 16;   // number of blocks
 int32_t tsS3PageCacheSize = 4096;  // number of pages
-int32_t tsS3UploadDelaySec = 60 * 60;
+int32_t tsS3UploadDelaySec = 60 * 60 * 24;
+
+bool    tsExperimental = true;
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -355,11 +358,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
   char cfgFile[PATH_MAX + 100] = {0};
 
   taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
+  char  lastC = cfgDir[strlen(cfgDir) - 1];
+  char *tdDirsep = TD_DIRSEP;
+  if (lastC == '\\' || lastC == '/') {
+    tdDirsep = "";
+  }
   if (taosIsDir(cfgDir)) {
 #ifdef CUS_PROMPT
-    snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT);
+    snprintf(cfgFile, sizeof(cfgFile),
+             "%s"
+             "%s"
+             "%s.cfg",
+             cfgDir, tdDirsep, CUS_PROMPT);
 #else
-    snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
+    snprintf(cfgFile, sizeof(cfgFile),
+             "%s"
+             "%s"
+             "taos.cfg",
+             cfgDir, tdDirsep);
 #endif
   } else {
     tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
@@ -432,6 +448,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   return 0;
 }
 
@@ -508,11 +525,13 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 2;
   tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
+
   if (tsNumOfTaskQueueThreads >= 50) {
     tsNumOfTaskQueueThreads = 50;
   }
   if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
     return -1;
+  if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
   return 0;
 }
@@ -692,6 +711,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
       0)
     return -1;
+  if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
+      0)
+    return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
       0)
     return -1;
@@ -725,6 +747,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
     return -1;
+
   if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
@@ -776,6 +799,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
+
   GRANT_CFG_ADD;
   return 0;
 }
@@ -954,6 +979,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32;
   metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32;
   stDebugFlag = cfgGetItem(pCfg, "stDebugFlag")->i32;
+  sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32;
 }
 
 static int32_t taosSetSlowLogScope(char *pScope) {
@@ -1060,6 +1086,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
 
   tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
+
+  tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
   return 0;
 }
 
@@ -1139,6 +1167,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
 
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
+  tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
   tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
   tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32;
@@ -1193,6 +1222,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
   tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
 
+  tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
+
   GRANT_CFG_GET;
   return 0;
 }
@@ -1269,7 +1300,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 static int32_t taosCheckGlobalCfg() {
   uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn);
   if (ipv4 == 0xffffffff) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    terrno = TSDB_CODE_RPC_FQDN_ERROR;
     uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr());
     return -1;
   }
@@ -1343,7 +1374,7 @@ void taosCleanupCfg() {
 
 typedef struct {
   const char *optionName;
-  void       *optionVar;
+  void *      optionVar;
 } OptionNameAndVar;
 
 static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) {
@@ -1356,7 +1387,7 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize,
     switch (pItem->dtype) {
       case CFG_DTYPE_BOOL: {
         int32_t flag = pItem->i32;
-        bool   *pVar = pOptions[d].optionVar;
+        bool *  pVar = pOptions[d].optionVar;
         uInfo("%s set from %d to %d", optName, *pVar, flag);
         *pVar = flag;
         terrno = TSDB_CODE_SUCCESS;
@@ -1428,7 +1459,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag},
         {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},     {"smaDebugFlag", &smaDebugFlag},
         {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},     {"metaDebugFlag", &metaDebugFlag},
-        {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag},
+        {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag},
     };
 
     static OptionNameAndVar options[] = {
@@ -1453,6 +1484,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"timeseriesThreshold", &tsTimeSeriesThreshold},
         {"tmqMaxTopicNum", &tmqMaxTopicNum},
         {"transPullupInterval", &tsTransPullupInterval},
+        {"compactPullupInterval", &tsCompactPullupInterval},
         {"trimVDbIntervalSec", &tsTrimVDbIntervalSec},
         {"ttlBatchDropNum", &tsTtlBatchDropNum},
         {"ttlFlushThreshold", &tsTtlFlushThreshold},
@@ -1462,6 +1494,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"s3PageCacheSize", &tsS3PageCacheSize},
         {"s3UploadDelaySec", &tsS3UploadDelaySec},
         {"supportVnodes", &tsNumOfSupportVnodes},
+        {"experimental", &tsExperimental}
     };
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@@ -1562,15 +1595,15 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
         matched = true;
       } else if (strcasecmp("minimalTmpDirGB", name) == 0) {
         tsTempSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
-        uInfo("%s set to %"PRId64, name, tsTempSpace.reserved);
+        uInfo("%s set to %" PRId64, name, tsTempSpace.reserved);
         matched = true;
       } else if (strcasecmp("minimalDataDirGB", name) == 0) {
         tsDataSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
-        uInfo("%s set to %"PRId64, name, tsDataSpace.reserved);
+        uInfo("%s set to %" PRId64, name, tsDataSpace.reserved);
         matched = true;
       } else if (strcasecmp("minimalLogDirGB", name) == 0) {
         tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024);
-        uInfo("%s set to %"PRId64, name, tsLogSpace.reserved);
+        uInfo("%s set to %" PRId64, name, tsLogSpace.reserved);
         matched = true;
       }
       break;
@@ -1685,6 +1718,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
         {"shellActivityTimer", &tsShellActivityTimer},
         {"slowLogThreshold", &tsSlowLogThreshold},
         {"useAdapter", &tsUseAdapter},
+        {"experimental", &tsExperimental}
     };
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@@ -1736,7 +1770,8 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) {
   taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite);
   taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite);
   taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite);
   uInfo("all debug flag are set to %d", flag);
 }
 
-int8_t taosGranted() { return atomic_load_8(&tsGrant); }
+int8_t taosGranted() { return atomic_load_8(&tsGrant); }

source/common/src/tmsg.c

@@ -3424,6 +3424,66 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
 
 void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
 
+int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->compactId) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->bAccepted) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->compactId) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->bAccepted) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  ENCODESQL();
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  DECODESQL();
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSKillCompactReq(SKillCompactReq *pReq) { FREESQL(); }
+
 int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
   if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
@@ -4259,6 +4319,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -4276,6 +4337,11 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI64(&decoder, &pReq->compactId) < 0) return -1;
+  } else {
+    pReq->compactId = -1;
+  }
 
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
@@ -5051,6 +5117,75 @@ int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
   return 0;
 }
 
+int32_t tSerializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, ((uint8_t *)buf) + headLen, bufLen - headLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numberFileset) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->finished) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -5139,6 +5274,8 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
   if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
 
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -5165,6 +5302,42 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
     if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;

source/common/src/ttime.c

@@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
     i += 2;
   }
 
+  if (hour > 12 || hour < 0) {
+    return -1;
+  }
+
   // return error if there're illegal charaters after min(2 Digits)
   char* minStr = &str[i];
   if (minStr[1] != '\0' && minStr[2] != '\0') {
@@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
   }
 
   int64_t minute = strnatoi(&str[i], 2);
-  if (minute > 59) {
+  if (minute > 59 || (hour == 12 && minute > 0)) {
     return -1;
   }
 
@@ -646,7 +650,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
 
   /* get the basic numeric value */
   int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
-  if (timestamp < 0 || errno != 0) {
+  if ((timestamp == 0 && token[0] != '0') || errno != 0) {
     return -1;
   }