other: merge main into 3.0

.gitignore

@@ -11,6 +11,7 @@ CMakeSettings.json
 cmake-build-debug/
 cmake-build-release/
 cscope.out
+cscope.files
 .DS_Store
 debug/
 release/

CMakeLists.txt

@@ -41,6 +41,7 @@ add_subdirectory(source)
 add_subdirectory(tools)
 add_subdirectory(utils)
 add_subdirectory(examples/c)
+add_subdirectory(tests)
 include(${TD_SUPPORT_DIR}/cmake.install)
 
 # docs 

Jenkinsfile2

@@ -393,7 +393,7 @@ pipeline {
                     agent{label " Mac_catalina "}
                     steps {
                         catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
-                            timeout(time: 20, unit: 'MINUTES'){
+                            timeout(time: 30, unit: 'MINUTES'){
                                 pre_test()
                                 pre_test_build_mac()
                             }

cmake/cmake.define

@@ -97,7 +97,15 @@ ENDIF()
 SET(JEMALLOC_ENABLED OFF)
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
+    IF (${CMAKE_BUILD_TYPE} MATCHES "Release")
+      MESSAGE("${Green} will build Release version! ${ColourReset}")
+      SET(COMMON_FLAGS "/W3 /D_WIN32 /DWIN32 /Zi- /O2 /GL /MD")
+
+    ELSE ()
+      MESSAGE("${Green} will build Debug version! ${ColourReset}")
       SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
+    ENDIF()
+
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
     # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
     #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
@@ -173,17 +181,17 @@ ELSE ()
         ENDIF()
         MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
 
-        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
-            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
-        ENDIF()
-
-        IF (COMPILER_SUPPORT_AVX512VL)
-            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
-            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
-        MESSAGE(STATUS "avx512vl supported by compiler")
-        ENDIF()
+#        IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512f -mavx512vbmi")
+#            MESSAGE(STATUS "avx512f/avx512bmi supported by compiler")
+#        ENDIF()
+#
+#        IF (COMPILER_SUPPORT_AVX512VL)
+#            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512vl")
+#            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx512vl")
+#        MESSAGE(STATUS "avx512vl supported by compiler")
+#        ENDIF()
     ENDIF()
 
     # build mode

cmake/cmake.options

@@ -151,6 +151,7 @@ IF(${BUILD_S3})
 
 IF(${BUILD_WITH_S3})
 
+add_definitions(-DUSE_S3)
 option(BUILD_WITH_COS "If build with cos" OFF)
 
 ELSE ()

contrib/CMakeLists.txt

@@ -284,7 +284,8 @@ if (${BUILD_WITH_ROCKSDB})
                 SET(CMAKE_BUILD_TYPE Release)
             endif()
         endif(${TD_LINUX})
-        MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+        MESSAGE(STATUS "ROCKSDB CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+        MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
 
         if(${TD_DARWIN})
             SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
@@ -296,8 +297,12 @@ if (${BUILD_WITH_ROCKSDB})
 
         if (${TD_WINDOWS})
             SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+            SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819")
             option(WITH_JNI "" OFF)
+            if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd")
+               message("Rocksdb build runtime lib use /MT or /MTd")
                option(WITH_MD_LIBRARY "build with MD" OFF)
+            endif()
             set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
         endif(${TD_WINDOWS})
 
@@ -328,9 +333,11 @@ if (${BUILD_WITH_ROCKSDB})
         )
     else() 
         if (NOT ${TD_LINUX})
-            MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+            MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
+            MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
             if(${TD_DARWIN})
                 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized")
+                SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=maybe-uninitialized")
             endif(${TD_DARWIN})
 
             if (${TD_DARWIN_ARM64})
@@ -339,8 +346,12 @@ if (${BUILD_WITH_ROCKSDB})
 
             if (${TD_WINDOWS})
                 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819")
+                SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819")
                 option(WITH_JNI "" OFF)
+                if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd")
+                    message("Rocksdb build runtime lib use /MT or /MTd")
                     option(WITH_MD_LIBRARY "build with MD" OFF)
+                endif()
                 set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
             endif(${TD_WINDOWS})
 
@@ -406,7 +417,9 @@ endif()
 
 # pthread
 if(${BUILD_PTHREAD})
-    set(CMAKE_BUILD_TYPE debug)
+    if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+    SET(CMAKE_BUILD_TYPE Release)
+    endif()
     add_definitions(-DPTW32_STATIC_LIB)
     add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
@@ -554,13 +567,18 @@ if(${BUILD_GEOS})
     if(${TD_LINUX})
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}")
-        IF ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+        if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
             SET(CMAKE_BUILD_TYPE Release)
         endif()
     endif(${TD_LINUX})
     option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF)
     add_subdirectory(geos EXCLUDE_FROM_ALL)
+    if (${TD_WINDOWS})
+        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+        SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+    else ()
         unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD
+    endif(${TD_WINDOWS})
     target_include_directories(
         geos_c
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/geos/include>

docs/en/07-develop/01-connect/index.md

@@ -177,7 +177,7 @@ Just need to add the reference to [TDengine.Connector](https://www.nuget.org/pac
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.0" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.0" />
   </ItemGroup>
 
 </Project>

docs/en/07-develop/04-query-data/_cs_async.mdx

@@ -1,3 +0,0 @@
-```csharp
-{{#include docs/examples/csharp/asyncQuery/Program.cs}}
-```

docs/en/07-develop/04-query-data/index.mdx

@@ -15,7 +15,6 @@ import CQuery from "./_c.mdx";
 import PhpQuery from "./_php.mdx";
 import PyAsync from "./_py_async.mdx";
 import NodeAsync from "./_js_async.mdx";
-import CsAsync from "./_cs_async.mdx";
 import CAsync from "./_c_async.mdx";
 
 ## Introduction
@@ -174,9 +173,6 @@ Please note that async query can only be used with a native connection.
   <TabItem label="Python" value="python">
     <PyAsync />
   </TabItem>
-  <TabItem label="C#" value="csharp">
-    <CsAsync />
-  </TabItem>
   <TabItem label="C" value="c">
     <CAsync />
   </TabItem>

docs/en/07-develop/07-tmq.mdx

@@ -249,23 +249,23 @@ function close()
 <TabItem value="C#" label="C#">
 
 ```csharp
+class ConsumerBuilder<TValue>
+
 ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
 
-virtual IConsumer Build()
-
-Consumer(ConsumerBuilder builder)
+public IConsumer<TValue> Build()
 
 void Subscribe(IEnumerable<string> topics)
 
 void Subscribe(string topic) 
 
-ConsumeResult Consume(int millisecondsTimeout)
+ConsumeResult<TValue> Consume(int millisecondsTimeout)
 
 List<string> Subscription()
 
 void Unsubscribe()
  
-void Commit(ConsumeResult consumerResult)
+List<TopicPartitionOffset> Commit()
 
 void Close()
 ```
@@ -501,25 +501,19 @@ let consumer = taos.consumer({
 <TabItem value="C#" label="C#">
 
 ```csharp
-using TDengineTMQ;
-
-// Create consumer groups on demand (GourpID) and enable automatic commits (EnableAutoCommit),
-// an automatic commit interval (AutoCommitIntervalMs), and a username (TDConnectUser) and password (TDConnectPasswd)
-var cfg = new ConsumerConfig
+var cfg = new Dictionary<string, string>()
 {
-    EnableAutoCommit = "true"
-    AutoCommitIntervalMs = "1000"
-    GourpId = "TDengine-TMQ-C#",
-    TDConnectUser = "root",
-    TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "latest"
-    MsgWithTableName = "true",
-    TDConnectIp = "127.0.0.1",
-    TDConnectPort = "6030"
+    { "group.id", "group1" },
+    { "auto.offset.reset", "latest" },
+    { "td.connect.ip", "127.0.0.1" },
+    { "td.connect.user", "root" },
+    { "td.connect.pass", "taosdata" },
+    { "td.connect.port", "6030" },
+    { "client.id", "tmq_example" },
+    { "enable.auto.commit", "true" },
+    { "msg.with.table.name", "false" },
 };
-
-var consumer = new ConsumerBuilder(cfg).Build();
-
+var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
 ```
 
 </TabItem>
@@ -748,10 +742,12 @@ while(true){
 ## Consume data
 while (true)
 {
-    var consumerRes = consumer.Consume(100);
-    // process ConsumeResult
-    ProcessMsg(consumerRes);
-    consumer.Commit(consumerRes);
+    using (var result = consumer.Consume(500))
+    {
+        if (result == null) continue;
+        ProcessMsg(result);
+        consumer.Commit();
+    }
 }
 ```
 

docs/en/08-client-libraries/05-go.mdx

@@ -74,7 +74,7 @@ If it is a TDengine error, you can get the error code and error information in t
 ### Pre-installation preparation
 
 * Install Go development environment (Go 1.14 and above, GCC 4.8.5 and above)
-* If you use the native connection, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for specific steps
+* If you use the native connector, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for specific steps
 
 Configure the environment variables and check the command.
 

docs/en/12-taos-sql/05-insert.md

@@ -159,7 +159,7 @@ Automatically creating table and the table name is specified through the `tbname
 ```sql
 INSERT INTO meters(tbname, location, groupId, ts, current, phase) 
                 values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) 
-                values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
-                values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
+                ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
+                ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
 ```
 

docs/en/12-taos-sql/26-udf.md

@@ -53,7 +53,7 @@ CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [
   CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 64;
   ``` 
 
-For more information about user-defined functions, see [User-Defined Functions](/develop/udf).
+For more information about user-defined functions, see [User-Defined Functions](../../develop/udf).
 
 ## Manage UDF
 

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -262,6 +262,63 @@ The following types may be returned:
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` and `GEOMETRY` types return data as Hex string, example:
+
+Prepare data
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+Execute query
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+Return results
+
+```json
+{
+     "code": 0,
+     "column_meta": [
+         [
+             "ts",
+             "TIMESTAMP",
+             8
+         ],
+         [
+             "c1",
+             "VARBINARY",
+             20
+         ],
+         [
+             "c2",
+             "GEOMETRY",
+             100
+         ]
+     ],
+     "data": [
+         [
+             "2023-11-01T06:28:15.210Z",
+             "7f8290",
+             "010100000000000000000059400000000000005940"
+         ]
+     ],
+     "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` is [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/) format for `point(100 100)`
 
 #### Errors
 

docs/examples/csharp/.gitignore

@@ -1,5 +1,5 @@
 .vs
-asyncQuery/bin
+.idea
 connect/bin
 influxdbLine/bin
 optsJSON/bin
@@ -12,7 +12,6 @@ wsConnect/bin
 wsInsert/bin
 wsQuery/bin
 wsStmt/bin
-asyncQuery/obj
 connect/obj
 influxdbLine/obj
 optsJSON/obj

docs/examples/csharp/asyncQuery/Program.cs

@@ -1,111 +0,0 @@
-using System;
-using System.Collections.Generic;
-using TDengineDriver;
-using TDengineDriver.Impl;
-using System.Runtime.InteropServices;
-
-namespace TDengineExample
-{
-    public class AsyncQueryExample
-    {
-        static void Main()
-        {
-            IntPtr conn = GetConnection();
-            try
-            {
-                QueryAsyncCallback queryAsyncCallback = new QueryAsyncCallback(QueryCallback);
-                TDengine.QueryAsync(conn, "select * from meters", queryAsyncCallback, IntPtr.Zero);
-                Thread.Sleep(2000);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static void QueryCallback(IntPtr param, IntPtr taosRes, int code)
-        {
-            if (code == 0 && taosRes != IntPtr.Zero)
-            {
-                FetchRawBlockAsyncCallback fetchRowAsyncCallback = new FetchRawBlockAsyncCallback(FetchRawBlockCallback);
-                TDengine.FetchRawBlockAsync(taosRes, fetchRowAsyncCallback, param);
-            }
-            else
-            {
-                throw new Exception($"async query data failed,code:{code},reason:{TDengine.Error(taosRes)}");
-            }
-        }
-
-        // Iteratively call this interface until "numOfRows" is no greater than 0.
-        static void FetchRawBlockCallback(IntPtr param, IntPtr taosRes, int numOfRows)
-        {
-            if (numOfRows > 0)
-            {
-                Console.WriteLine($"{numOfRows} rows async retrieved");
-                IntPtr pdata = TDengine.GetRawBlock(taosRes);
-                List<TDengineMeta> metaList = TDengine.FetchFields(taosRes);
-                List<object> dataList = LibTaos.ReadRawBlock(pdata, metaList, numOfRows);
-
-                for (int i = 0; i < dataList.Count; i++)
-                {
-                    if (i != 0 && (i + 1) % metaList.Count == 0)
-                    {
-                        Console.WriteLine("{0}\t|", dataList[i]);
-                    }
-                    else
-                    {
-                        Console.Write("{0}\t|", dataList[i]);
-                    }
-                }
-                Console.WriteLine("");
-                TDengine.FetchRawBlockAsync(taosRes, FetchRawBlockCallback, param);
-            }
-            else
-            {
-                if (numOfRows == 0)
-                {
-                    Console.WriteLine("async retrieve complete.");
-                }
-                else
-                {
-                    throw new Exception($"FetchRawBlockCallback callback error, error code {numOfRows}");
-                }
-                TDengine.FreeResult(taosRes);
-            }
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-    }
-}
-
-// //output:
-// // Connect to TDengine success
-// // 8 rows async retrieved
-
-// // 1538548685500   |       11.8    |       221     |       0.28    |       california.losangeles   |       2       |
-// // 1538548696600   |       13.4    |       223     |       0.29    |       california.losangeles   |       2       |
-// // 1538548685000   |       10.8    |       223     |       0.29    |       california.losangeles   |       3       |
-// // 1538548686500   |       11.5    |       221     |       0.35    |       california.losangeles   |       3       |
-// // 1538548685000   |       10.3    |       219     |       0.31    |       california.sanfrancisco         |       2       |
-// // 1538548695000   |       12.6    |       218     |       0.33    |       california.sanfrancisco         |       2       |
-// // 1538548696800   |       12.3    |       221     |       0.31    |       california.sanfrancisco         |       2       |
-// // 1538548696650   |       10.3    |       218     |       0.25    |       california.sanfrancisco         |       3       |
-// // async retrieve complete.

docs/examples/csharp/asyncQuery/asyncquery.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <Nullable>enable</Nullable>
-    <StartupObject>TDengineExample.AsyncQueryExample</StartupObject>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
-  </ItemGroup>
-
-</Project>

docs/examples/csharp/connect/Program.cs

@@ -1,4 +1,5 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
@@ -7,23 +8,11 @@ namespace TDengineExample
     {
         static void Main(String[] args)
         {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            TDengine.Close(conn);
-            TDengine.Cleanup();
+                Console.WriteLine("connected");
+            }
         }
     }
 }

docs/examples/csharp/connect/connect.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/csharp.sln

@@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.30114.105
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "asyncquery", "asyncQuery\asyncquery.csproj", "{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "connect", "connect\connect.csproj", "{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "influxdbline", "influxdbLine\influxdbline.csproj", "{6A24FB80-1E3C-4E2D-A5AB-914FA583874D}"
@@ -38,10 +36,6 @@ Global
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E2A5F00C-14E7-40E1-A2DE-6AB2975616D3}.Release|Any CPU.Build.0 = Release|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CCC5042D-93FC-4AE0-B2F6-7E692FD476B7}.Release|Any CPU.ActiveCfg = Release|Any CPU

docs/examples/csharp/influxdbLine/Program.cs

@@ -1,4 +1,5 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
@@ -6,60 +7,23 @@ namespace TDengineExample
     {
         static void Main()
         {
-            IntPtr conn = GetConnection();
-            PrepareDatabase(conn);
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
+            {
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("use test");
                 string[] lines = {
                     "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=0.28 1648432611249",
                     "meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0.29 1648432611250",
                     "meters,location=California.LosAngeles,groupid=3 current=10.8,voltage=223,phase=0.29 1648432611249",
                     "meters,location=California.LosAngeles,groupid=3 current=11.3,voltage=221,phase=0.35 1648432611250"
                 };
-            IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS);
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
+                client.SchemalessInsert(lines,
+                    TDengineSchemalessProtocol.TSDB_SML_LINE_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
-            else
-            {
-                int affectedRows = TDengine.AffectRows(res);
-                Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-            }
-            TDengine.FreeResult(res);
-
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
         }
     }
 
 }
-
-}

docs/examples/csharp/influxdbLine/influxdbline.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/optsJSON/Program.cs

@@ -1,68 +1,27 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class OptsJsonExample
     {
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                PrepareDatabase(conn);
-                string[] lines = { "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("use test");
+                string[] lines =
+                {
+                    "[{\"metric\": \"meters.current\", \"timestamp\": 1648432611249, \"value\": 10.3, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
                     " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611249, \"value\": 219, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}, " +
                     "{\"metric\": \"meters.current\", \"timestamp\": 1648432611250, \"value\": 12.6, \"tags\": {\"location\": \"California.SanFrancisco\", \"groupid\": 2}}," +
                     " {\"metric\": \"meters.voltage\", \"timestamp\": 1648432611250, \"value\": 221, \"tags\": {\"location\": \"California.LosAngeles\", \"groupid\": 1}}]"
                 };
-
-                IntPtr res = TDengine.SchemalessInsert(conn, lines, 1, (int)TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
-                if (TDengine.ErrorNo(res) != 0)
-                {
-                    throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
-                }
-                else
-                {
-                    int affectedRows = TDengine.AffectRows(res);
-                    Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-                }
-                TDengine.FreeResult(res);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
+                client.SchemalessInsert(lines, TDengineSchemalessProtocol.TSDB_SML_JSON_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
         }
     }

docs/examples/csharp/optsJSON/optsJSON.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/optsTelnet/Program.cs

@@ -1,15 +1,18 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class OptsTelnetExample
     {
-        static void Main()
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            try
+            var builder =
+                new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                PrepareDatabase(conn);
+                client.Exec("CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
+                client.Exec("USE test");
                 string[] lines = {
                     "meters.current 1648432611249 10.3 location=California.SanFrancisco groupid=2",
                     "meters.current 1648432611250 12.6 location=California.SanFrancisco groupid=2",
@@ -20,53 +23,9 @@ namespace TDengineExample
                     "meters.voltage 1648432611249 221 location=California.LosAngeles groupid=3",
                     "meters.voltage 1648432611250 217 location=California.LosAngeles groupid=3",
                 };
-                IntPtr res = TDengine.SchemalessInsert(conn, lines, lines.Length, (int)TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL, (int)TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_NOT_CONFIGURED);
-                if (TDengine.ErrorNo(res) != 0)
-                {
-                    throw new Exception("SchemalessInsert failed since " + TDengine.Error(res));
-                }
-                else
-                {
-                    int affectedRows = TDengine.AffectRows(res);
-                    Console.WriteLine($"SchemalessInsert success, affected {affectedRows} rows");
-                }
-                TDengine.FreeResult(res);
-            }
-            catch
-            {
-                TDengine.Close(conn);
-            }
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareDatabase(IntPtr conn)
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE test WAL_RETENTION_PERIOD 3600");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to create database, reason: " + TDengine.Error(res));
-            }
-            res = TDengine.Query(conn, "USE test");
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception("failed to change database, reason: " + TDengine.Error(res));
+                client.SchemalessInsert(lines,
+                    TDengineSchemalessProtocol.TSDB_SML_TELNET_PROTOCOL,
+                    TDengineSchemalessPrecision.TSDB_SML_TIMESTAMP_MILLI_SECONDS, 0, ReqId.GetReqId());
             }
         }
     }

docs/examples/csharp/optsTelnet/optstelnet.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/query/Program.cs

@@ -1,80 +1,35 @@
-using TDengineDriver;
-using TDengineDriver.Impl;
-using System.Runtime.InteropServices;
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class QueryExample
     {
-        static void Main()
+        public static void Main(string[] args)
+        {
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-            IntPtr conn = GetConnection();
                 try
                 {
-                // run query
-                IntPtr res = TDengine.Query(conn, "SELECT * FROM meters LIMIT 2");
-                if (TDengine.ErrorNo(res) != 0)
+                    client.Exec("use power");
+                    string query = "SELECT * FROM meters";
+                    using (var rows = client.Query(query))
                     {
-                    throw new Exception("Failed to query since: " + TDengine.Error(res));
-                }
-
-                // get filed count
-                int fieldCount = TDengine.FieldCount(res);
-                Console.WriteLine("fieldCount=" + fieldCount);
-
-                // print column names
-                List<TDengineMeta> metas = LibTaos.GetMeta(res);
-                for (int i = 0; i < metas.Count; i++)
+                        while (rows.Read())
                         {
-                    Console.Write(metas[i].name + "\t");
+                            Console.WriteLine(
+                                $"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
                         }
-                Console.WriteLine();
-
-                // print values
-                List<Object> resData = LibTaos.GetData(res);
-                for (int i = 0; i < resData.Count; i++)
+                    }
+                }
+                catch (Exception e)
                 {
-                    Console.Write($"|{resData[i].ToString()} \t");
-                    if (((i + 1) % metas.Count == 0))
-                    {
-                        Console.WriteLine("");
-                    }
-                }
-                Console.WriteLine();
-
-                // Free result after use
-                TDengine.FreeResult(res);
-            }
-            finally
-            {
-                TDengine.Close(conn);
-            }
-
-        }
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
+            }
         }
     }
 }
-
-// output:
-// Connect to TDengine success
-// fieldCount=6
-// ts      current voltage phase   location        groupid
-// 1648432611249   10.3    219     0.31    California.SanFrancisco        2
-// 1648432611749   12.6    218     0.33    California.SanFrancisco        2

docs/examples/csharp/query/query.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/sqlInsert/Program.cs

@@ -1,69 +1,47 @@
-using TDengineDriver;
-
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class SQLInsertExample
     {
 
-        static void Main()
+        public static void Main(string[] args)
+        {
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-            IntPtr conn = GetConnection();
                 try
                 {
-                IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
-                CheckRes(conn, res, "failed to create database");
-                res = TDengine.Query(conn, "USE power");
-                CheckRes(conn, res, "failed to change database");
-                res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
-                CheckRes(conn, res, "failed to create stable");
-                var sql = "INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
-                            "d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
-                            "d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
-                            "d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
-                res = TDengine.Query(conn, sql);
-                CheckRes(conn, res, "failed to insert data");
-                int affectedRows = TDengine.AffectRows(res);
-                Console.WriteLine("affectedRows " + affectedRows);
-                TDengine.FreeResult(res);
+                    client.Exec("create database power");
+                    client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    string insertQuery =
+                        "INSERT INTO " +
+                        "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
+                        "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
+                        "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
+                        "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
+                        "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
+                        "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
+                        "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
+                        "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
+                    client.Exec(insertQuery);
                 }
-            finally
+                catch (Exception e)
                 {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void CheckRes(IntPtr conn, IntPtr res, String errorMsg)
-        {
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception($"{errorMsg} since: {TDengine.Error(res)}");
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
             }
         }
-
     }
 }
-
-// output:
-// Connect to TDengine success
-// affectedRows 8

docs/examples/csharp/sqlInsert/sqlinsert.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/stmtInsert/Program.cs

@@ -1,109 +1,38 @@
-using TDengineDriver;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace TDengineExample
 {
     internal class StmtInsertExample
     {
-        private static IntPtr conn;
-        private static IntPtr stmt;
-        static void Main()
+        public static void Main(string[] args)
+        {
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-            conn = GetConnection();
                 try
                 {
-                PrepareSTable();
-                // 1. init and prepare
-                stmt = TDengine.StmtInit(conn);
-                if (stmt == IntPtr.Zero)
+                    client.Exec($"create database power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    using (var stmt = client.StmtInit())
                     {
-                    throw new Exception("failed to init stmt.");
+                        stmt.Prepare(
+                            "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
+                        var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
+                        stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
+                        stmt.AddBatch();
+                        stmt.Exec();
+                        var affected = stmt.Affected();
+                        Console.WriteLine($"affected rows: {affected}");
                     }
-                int res = TDengine.StmtPrepare(stmt, "INSERT INTO ? USING meters TAGS(?, ?) VALUES(?, ?, ?, ?)");
-                CheckStmtRes(res, "failed to prepare stmt");
-
-                // 2. bind table name and tags
-                TAOS_MULTI_BIND[] tags = new TAOS_MULTI_BIND[2] { TaosMultiBind.MultiBindBinary(new string[] { "California.SanFrancisco" }), TaosMultiBind.MultiBindInt(new int?[] { 2 }) };
-                res = TDengine.StmtSetTbnameTags(stmt, "d1001", tags);
-                CheckStmtRes(res, "failed to bind table name and tags");
-
-                // 3. bind values
-                TAOS_MULTI_BIND[] values = new TAOS_MULTI_BIND[4] {
-                TaosMultiBind.MultiBindTimestamp(new long[2] { 1648432611249, 1648432611749}),
-                TaosMultiBind.MultiBindFloat(new float?[2] { 10.3f, 12.6f}),
-                TaosMultiBind.MultiBindInt(new int?[2] { 219, 218}),
-                TaosMultiBind.MultiBindFloat(new float?[2]{ 0.31f, 0.33f})
-            };
-                res = TDengine.StmtBindParamBatch(stmt, values);
-                CheckStmtRes(res, "failed to bind params");
-
-                // 4. add batch
-                res = TDengine.StmtAddBatch(stmt);
-                CheckStmtRes(res, "failed to add batch");
-
-                // 5. execute
-                res = TDengine.StmtExecute(stmt);
-                CheckStmtRes(res, "failed to execute");
-
-                // 6. free 
-                TaosMultiBind.FreeTaosBind(tags);
-                TaosMultiBind.FreeTaosBind(values);
                 }
-            finally
+                catch (Exception e)
                 {
-                TDengine.Close(conn);
-            }
-
-        }
-
-        static IntPtr GetConnection()
-        {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
-            {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
-            {
-                Console.WriteLine("Connect to TDengine success");
-            }
-            return conn;
-        }
-
-        static void PrepareSTable()
-        {
-            IntPtr res = TDengine.Query(conn, "CREATE DATABASE power WAL_RETENTION_PERIOD 3600");
-            CheckResPtr(res, "failed to create database");
-            res = TDengine.Query(conn, "USE power");
-            CheckResPtr(res, "failed to change database");
-            res = TDengine.Query(conn, "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT)");
-            CheckResPtr(res, "failed to create stable");
-        }
-
-        static void CheckStmtRes(int res, string errorMsg)
-        {
-            if (res != 0)
-            {
-                Console.WriteLine(errorMsg + ", " + TDengine.StmtErrorStr(stmt));
-                int code = TDengine.StmtClose(stmt);
-                if (code != 0)
-                {
-                    throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} ");
+                    Console.WriteLine(e);
+                    throw;
                 }
             }
         }
-
-        static void CheckResPtr(IntPtr res, string errorMsg)
-        {
-            if (TDengine.ErrorNo(res) != 0)
-            {
-                throw new Exception(errorMsg + " since:" + TDengine.Error(res));
-            }
-        }
-
     }
 }

docs/examples/csharp/stmtInsert/stmtinsert.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/subscribe/Program.cs

@@ -1,95 +1,72 @@
-using System;
-using TDengineTMQ;
-using TDengineDriver;
-using System.Runtime.InteropServices;
+
+using TDengine.Driver;
+using TDengine.Driver.Client;
+using TDengine.TMQ;
 
 namespace TMQExample
 {
     internal class SubscribeDemo
     {
-        static void Main(string[] args)
+        public static void Main(string[] args)
         {
-            IntPtr conn = GetConnection();
-            string topic = "topic_example";
-            //create topic 
-            IntPtr res = TDengine.Query(conn, $"create topic if not exists {topic} as select * from meters");
-
-            if (TDengine.ErrorNo(res) != 0 )
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception($"create topic failed, reason:{TDengine.Error(res)}");
-            }
-
-            var cfg = new ConsumerConfig
+                try
                 {
-                GourpId = "group_1",
-                TDConnectUser = "root",
-                TDConnectPasswd = "taosdata",
-                MsgWithTableName = "true",
-                TDConnectIp = "127.0.0.1",
+                    client.Exec("CREATE DATABASE power");
+                    client.Exec("USE power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    client.Exec("CREATE TOPIC topic_meters as SELECT * from power.meters");
+                    var cfg = new Dictionary<string, string>()
+                    {
+                        { "group.id", "group1" },
+                        { "auto.offset.reset", "latest" },
+                        { "td.connect.ip", "127.0.0.1" },
+                        { "td.connect.user", "root" },
+                        { "td.connect.pass", "taosdata" },
+                        { "td.connect.port", "6030" },
+                        { "client.id", "tmq_example" },
+                        { "enable.auto.commit", "true" },
+                        { "msg.with.table.name", "false" },
                     };
-
-            // create consumer 
-            var consumer = new ConsumerBuilder(cfg)
-                .Build();
-
-            // subscribe
-            consumer.Subscribe(topic);
-
-            // consume 
-            for (int i = 0; i < 5; i++)
+                    var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
+                    consumer.Subscribe(new List<string>() { "topic_meters" });
+                    Task.Run(InsertData);
+                    while (true)
                     {
-                var consumeRes = consumer.Consume(300);
-                // print consumeResult
-                foreach (KeyValuePair<TopicPartition, TaosResult> kv in consumeRes.Message)
+                        using (var cr = consumer.Consume(500))
                         {
-                    Console.WriteLine("topic partitions:\n{0}", kv.Key.ToString());
-
-                    kv.Value.Metas.ForEach(meta =>
+                            if (cr == null) continue;
+                            foreach (var message in cr.Message)
                             {
-                        Console.Write("{0} {1}({2}) \t|", meta.name, meta.TypeName(), meta.size);
-                    });
-                    Console.WriteLine("");
-                    kv.Value.Datas.ForEach(data =>
+                                Console.WriteLine(
+                                    $"message {{{((DateTime)message.Value["ts"]).ToString("yyyy-MM-dd HH:mm:ss.fff")}, " +
+                                    $"{message.Value["current"]}, {message.Value["voltage"]}, {message.Value["phase"]}}}");
+                            }
+                        }
+                    }
+                }
+                catch (Exception e)
                 {
-                        Console.WriteLine(data.ToString());
-                    });
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
+            }
         }
 
-                consumer.Commit(consumeRes);
-                Console.WriteLine("\n================ {0} done ", i);
-
-            }
-
-            // retrieve topic list
-            List<string> topics = consumer.Subscription();
-            topics.ForEach(t => Console.WriteLine("topic name:{0}", t));
-
-            // unsubscribe
-            consumer.Unsubscribe();
-
-            // close consumer after use.Otherwise will lead memory leak.
-            consumer.Close();
-            TDengine.Close(conn);
-
-        }
-
-        static IntPtr GetConnection()
+        static void InsertData()
         {
-            string host = "localhost";
-            short port = 6030;
-            string username = "root";
-            string password = "taosdata";
-            string dbname = "power";
-            var conn = TDengine.Connect(host, username, password, dbname, port);
-            if (conn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("host=localhost;port=6030;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                throw new Exception("Connect to TDengine failed");
-            }
-            else
+                while (true)
                 {
-                Console.WriteLine("Connect to TDengine success");
+                    client.Exec("INSERT into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(now,11.5,219,0.30)");
+                    Task.Delay(1000).Wait();
+                }
             }
-            return conn;
         }
     }
 

docs/examples/csharp/subscribe/subscribe.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" />
   </ItemGroup>
 
 </Project>

docs/examples/csharp/wsConnect/Program.cs

@@ -1,28 +1,18 @@
 using System;
-using TDengineWS.Impl;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSConnExample
     {
-        static int Main(string[] args)
+        static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-  
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
-            }
-            else
-            {
-                Console.WriteLine("Establish connect success.");
-                // close connection.
-                LibTaosWS.WSClose(wsConn);
-            }
-
-            return 0;
+                Console.WriteLine("connected");
+            }
         }
     }
 }

docs/examples/csharp/wsConnect/wsConnect.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsInsert/Program.cs

@@ -1,61 +1,46 @@
 using System;
-using TDengineWS.Impl;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSInsertExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-
-            // Assert if connection is validate
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
-            }
-            else
+                try
                 {
-                Console.WriteLine("Establish connect success.");
+                    client.Exec("create database power");
+                    client.Exec("CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    string insertQuery =
+                        "INSERT INTO " +
+                        "power.d1001 USING power.meters TAGS(2,'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.30000, 219, 0.31000) " +
+                        "('2023-10-03 14:38:15.000', 12.60000, 218, 0.33000) " +
+                        "('2023-10-03 14:38:16.800', 12.30000, 221, 0.31000) " +
+                        "power.d1002 USING power.meters TAGS(3, 'California.SanFrancisco') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:16.650', 10.30000, 218, 0.25000) " +
+                        "power.d1003 USING power.meters TAGS(2,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.500', 11.80000, 221, 0.28000) " +
+                        "('2023-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
+                        "power.d1004 USING power.meters TAGS(3,'California.LosAngeles') " +
+                        "VALUES " +
+                        "('2023-10-03 14:38:05.000', 10.80000, 223, 0.29000) " +
+                        "('2023-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
+                    client.Exec(insertQuery);
                 }
-
-            string createTable = "CREATE STABLE test.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);";
-            string insert = "INSERT INTO test.d1001 USING test.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000)" +
-                            "test.d1002 USING test.meters TAGS('California.SanFrancisco', 3) VALUES('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000)" +
-                            "test.d1003 USING test.meters TAGS('California.LosAngeles', 2) VALUES('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000)('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) " +
-                            "test.d1004 USING test.meters TAGS('California.LosAngeles', 3) VALUES('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000)('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)";
-
-            IntPtr wsRes = LibTaosWS.WSQuery(wsConn, createTable);
-            ValidInsert("create table", wsRes);
-            LibTaosWS.WSFreeResult(wsRes);
-
-            wsRes = LibTaosWS.WSQuery(wsConn, insert);
-            ValidInsert("insert data", wsRes);
-            LibTaosWS.WSFreeResult(wsRes);
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
-        }
-
-        static void ValidInsert(string desc, IntPtr wsRes)
+                catch (Exception e)
                 {
-            int code = LibTaosWS.WSErrorNo(wsRes);
-            if (code != 0)
-            {
-                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
-            }
-            else
-            {
-                Console.WriteLine("{0} success affect {2} rows, cost {1} nanoseconds", desc, LibTaosWS.WSTakeTiming(wsRes), LibTaosWS.WSAffectRows(wsRes));
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
             }
         }
     }
-
 }
-// Establish connect success.
-// create table success affect 0 rows, cost 3717542 nanoseconds
-// insert data success affect 8 rows, cost 2613637 nanoseconds

docs/examples/csharp/wsInsert/wsInsert.csproj

@@ -6,7 +6,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsQuery/Program.cs

@@ -1,79 +1,36 @@
 using System;
-using TDengineWS.Impl;
-using System.Collections.Generic;
-using TDengineDriver;
+using System.Text;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSQueryExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-            if (wsConn == IntPtr.Zero)
+            var builder = new ConnectionStringBuilder("protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine("get WS connection failed");
-                return -1;
-            }
-            else
+                try
                 {
-                Console.WriteLine("Establish connect success.");
-            }
-
-            string select = "select * from test.meters";
-
-            // optional:wsRes = LibTaosWS.WSQuery(wsConn, select);
-            IntPtr wsRes = LibTaosWS.WSQueryTimeout(wsConn, select, 1);
-            // Assert if query execute success.
-            int code = LibTaosWS.WSErrorNo(wsRes);
-            if (code != 0)
+                    client.Exec("use power");
+                    string query = "SELECT * FROM meters";
+                    using (var rows = client.Query(query))
                     {
-                Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}");
-                LibTaosWS.WSFreeResult(wsRes);
-                return -1;
-            }
-
-            // get meta data
-            List<TDengineMeta> metas = LibTaosWS.WSGetFields(wsRes);
-            // get retrieved data
-            List<object> dataSet = LibTaosWS.WSGetData(wsRes);
-
-            // do something with result.
-            foreach (var meta in metas)
+                        while (rows.Read())
                         {
-                Console.Write("{0} {1}({2}) \t|\t", meta.name, meta.TypeName(), meta.size);
+                            Console.WriteLine(
+                                $"{((DateTime)rows.GetValue(0)):yyyy-MM-dd HH:mm:ss.fff}, {rows.GetValue(1)}, {rows.GetValue(2)}, {rows.GetValue(3)}, {rows.GetValue(4)}, {Encoding.UTF8.GetString((byte[])rows.GetValue(5))}");
                         }
-            Console.WriteLine("");
-
-            for (int i = 0; i < dataSet.Count;)
+                    }
+                }
+                catch (Exception e)
                 {
-                for (int j = 0; j < metas.Count; j++)
-                {
-                    Console.Write("{0}\t|\t", dataSet[i]);
-                    i++;
-                }
-                Console.WriteLine("");
-            }
-
-            // Free result after use.
-            LibTaosWS.WSFreeResult(wsRes);
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
+                    Console.WriteLine(e.ToString());
+                    throw;
+                }
+            }
         }
     }
 }
-
-// Establish connect success.
-// ts TIMESTAMP(8)         |       current FLOAT(4)        |       voltage INT(4)  |       phase FLOAT(4)  |       location BINARY(64)     |       groupid INT(4)  |
-// 1538548685000   |       10.8    |       223     |       0.29    |       California.LosAngeles   |       3       |
-// 1538548686500   |       11.5    |       221     |       0.35    |       California.LosAngeles   |       3       |
-// 1538548685500   |       11.8    |       221     |       0.28    |       California.LosAngeles   |       2       |
-// 1538548696600   |       13.4    |       223     |       0.29    |       California.LosAngeles   |       2       |
-// 1538548685000   |       10.3    |       219     |       0.31    |       California.SanFrancisco |       2       |
-// 1538548695000   |       12.6    |       218     |       0.33    |       California.SanFrancisco |       2       |
-// 1538548696800   |       12.3    |       221     |       0.31    |       California.SanFrancisco |       2       |
-// 1538548696650   |       10.3    |       218     |       0.25    |       California.SanFrancisco |       3       |

docs/examples/csharp/wsQuery/wsQuery.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/examples/csharp/wsStmt/Program.cs

@@ -1,98 +1,41 @@
 using System;
-using TDengineWS.Impl;
-using TDengineDriver;
-using System.Runtime.InteropServices;
+using TDengine.Driver;
+using TDengine.Driver.Client;
 
 namespace Examples
 {
     public class WSStmtExample
     {
-        static int Main(string[] args)
+        public static void Main(string[] args)
         {
-            const string DSN = "ws://root:taosdata@127.0.0.1:6041/test";
-            const string table = "meters";
-            const string database = "test";
-            const string childTable = "d1005";
-            string insert = $"insert into ? using {database}.{table} tags(?,?) values(?,?,?,?)";
-            const int numOfTags = 2;
-            const int numOfColumns = 4;
-
-            // Establish connection
-            IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN);
-            if (wsConn == IntPtr.Zero)
+            var builder =
+                new ConnectionStringBuilder(
+                    "protocol=WebSocket;host=localhost;port=6041;useSSL=false;username=root;password=taosdata");
+            using (var client = DbDriver.Open(builder))
             {
-                Console.WriteLine($"get WS connection failed");
-                return -1;
-            }
-            else
+                try
                 {
-                Console.WriteLine("Establish connect success...");
-            }
-
-            // init stmt
-            IntPtr wsStmt = LibTaosWS.WSStmtInit(wsConn);
-            if (wsStmt != IntPtr.Zero)
+                    client.Exec($"create database power");
+                    client.Exec(
+                        "CREATE STABLE power.meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (groupId INT, location BINARY(24))");
+                    using (var stmt = client.StmtInit())
                     {
-                int code = LibTaosWS.WSStmtPrepare(wsStmt, insert);
-                ValidStmtStep(code, wsStmt, "WSStmtPrepare");
-
-                TAOS_MULTI_BIND[] wsTags = new TAOS_MULTI_BIND[] { WSMultiBind.WSBindNchar(new string[] { "California.SanDiego" }), WSMultiBind.WSBindInt(new int?[] { 4 }) };
-                code = LibTaosWS.WSStmtSetTbnameTags(wsStmt, $"{database}.{childTable}", wsTags, numOfTags);
-                ValidStmtStep(code, wsStmt, "WSStmtSetTbnameTags");
-
-                TAOS_MULTI_BIND[] data = new TAOS_MULTI_BIND[4];
-                data[0] = WSMultiBind.WSBindTimestamp(new long[] { 1538548687000, 1538548688000, 1538548689000, 1538548690000, 1538548691000 });
-                data[1] = WSMultiBind.WSBindFloat(new float?[] { 10.30F, 10.40F, 10.50F, 10.60F, 10.70F });
-                data[2] = WSMultiBind.WSBindInt(new int?[] { 223, 221, 222, 220, 219 });
-                data[3] = WSMultiBind.WSBindFloat(new float?[] { 0.31F, 0.32F, 0.33F, 0.35F, 0.28F });
-                code = LibTaosWS.WSStmtBindParamBatch(wsStmt, data, numOfColumns);
-                ValidStmtStep(code, wsStmt, "WSStmtBindParamBatch");
-
-                code = LibTaosWS.WSStmtAddBatch(wsStmt);
-                ValidStmtStep(code, wsStmt, "WSStmtAddBatch");
-
-                IntPtr stmtAffectRowPtr = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(Int32)));
-                code = LibTaosWS.WSStmtExecute(wsStmt, stmtAffectRowPtr);
-                ValidStmtStep(code, wsStmt, "WSStmtExecute");
-                Console.WriteLine("WS STMT insert {0} rows...", Marshal.ReadInt32(stmtAffectRowPtr));
-                Marshal.FreeHGlobal(stmtAffectRowPtr);
-
-                LibTaosWS.WSStmtClose(wsStmt);
-
-                // Free unmanaged memory
-                WSMultiBind.WSFreeTaosBind(wsTags);
-                WSMultiBind.WSFreeTaosBind(data);
-
-                //check result with SQL "SELECT * FROM test.d1005;"
+                        stmt.Prepare(
+                            "Insert into power.d1001 using power.meters tags(2,'California.SanFrancisco') values(?,?,?,?)");
+                        var ts = new DateTime(2023, 10, 03, 14, 38, 05, 000);
+                        stmt.BindRow(new object[] { ts, (float)10.30000, (int)219, (float)0.31000 });
+                        stmt.AddBatch();
+                        stmt.Exec();
+                        var affected = stmt.Affected();
+                        Console.WriteLine($"affected rows: {affected}");
                     }
-            else
+                }
+                catch (Exception e)
                 {
-                Console.WriteLine("Init STMT failed...");
-            }
-
-            // close connection.
-            LibTaosWS.WSClose(wsConn);
-
-            return 0;
-        }
-
-        static void ValidStmtStep(int code, IntPtr wsStmt, string desc)
-        {
-            if (code != 0)
-            {
-                Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}");
-            }
-            else
-            {
-                Console.WriteLine("{0} success...", desc);
+                    Console.WriteLine(e);
+                    throw;
+                }
             }
         }
     }
 }
-
-// WSStmtPrepare success...
-// WSStmtSetTbnameTags success...
-// WSStmtBindParamBatch success...
-// WSStmtAddBatch success...
-// WSStmtExecute success...
-// WS STMT insert 5 rows...

docs/examples/csharp/wsStmt/wsStmt.csproj

@@ -7,7 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.*" GeneratePathProperty="true" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.*" GeneratePathProperty="true" />
   </ItemGroup>
   <Target Name="copyDLLDependency" BeforeTargets="BeforeBuild">
     <ItemGroup>

docs/zh/07-develop/01-connect/index.md

@@ -176,7 +176,7 @@ npm install @tdengine/rest
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="TDengine.Connector" Version="3.0.0" />
+    <PackageReference Include="TDengine.Connector" Version="3.1.0" />
   </ItemGroup>
 
 </Project>

docs/zh/07-develop/04-query-data/_cs_async.mdx

@@ -1,3 +0,0 @@
-```csharp
-{{#include docs/examples/csharp/asyncQuery/Program.cs}}
-```

docs/zh/07-develop/04-query-data/index.mdx

@@ -16,7 +16,6 @@ import CQuery from "./_c.mdx";
 import PhpQuery from "./_php.mdx";
 import PyAsync from "./_py_async.mdx";
 import NodeAsync from "./_js_async.mdx";
-import CsAsync from "./_cs_async.mdx";
 import CAsync from "./_c_async.mdx";
 
 ## 主要查询功能
@@ -175,9 +174,6 @@ Query OK, 6 rows in database (0.005515s)
   <TabItem label="Python" value="python">
     <PyAsync />
   </TabItem>
-  <TabItem label="C#" value="csharp">
-    <CsAsync />
-  </TabItem>
   <TabItem label="C" value="c">
     <CAsync />
   </TabItem>

docs/zh/07-develop/07-tmq.mdx

@@ -249,23 +249,23 @@ function close()
 <TabItem value="C#" label="C#">
 
 ```csharp
+class ConsumerBuilder<TValue>
+
 ConsumerBuilder(IEnumerable<KeyValuePair<string, string>> config)
 
-virtual IConsumer Build()
-
-Consumer(ConsumerBuilder builder)
+public IConsumer<TValue> Build()
 
 void Subscribe(IEnumerable<string> topics)
 
 void Subscribe(string topic) 
 
-ConsumeResult Consume(int millisecondsTimeout)
+ConsumeResult<TValue> Consume(int millisecondsTimeout)
 
 List<string> Subscription()
 
 void Unsubscribe()
  
-void Commit(ConsumeResult consumerResult)
+List<TopicPartitionOffset> Commit()
 
 void Close()
 ```
@@ -502,25 +502,19 @@ let consumer = taos.consumer({
 <TabItem value="C#" label="C#">
 
 ```csharp
-using TDengineTMQ;
-
-// 根据需要，设置消费组 (GourpId)、自动提交 (EnableAutoCommit)、
-// 自动提交时间间隔 (AutoCommitIntervalMs)、用户名 (TDConnectUser)、密码 (TDConnectPasswd) 等参数
-var cfg = new ConsumerConfig
+var cfg = new Dictionary<string, string>()
 {
-    EnableAutoCommit = "true"
-    AutoCommitIntervalMs = "1000"
-    GourpId = "TDengine-TMQ-C#",
-    TDConnectUser = "root",
-    TDConnectPasswd = "taosdata",
-    AutoOffsetReset = "latest"
-    MsgWithTableName = "true",
-    TDConnectIp = "127.0.0.1",
-    TDConnectPort = "6030"
+    { "group.id", "group1" },
+    { "auto.offset.reset", "latest" },
+    { "td.connect.ip", "127.0.0.1" },
+    { "td.connect.user", "root" },
+    { "td.connect.pass", "taosdata" },
+    { "td.connect.port", "6030" },
+    { "client.id", "tmq_example" },
+    { "enable.auto.commit", "true" },
+    { "msg.with.table.name", "false" },
 };
-
-var consumer = new ConsumerBuilder(cfg).Build();
-
+var consumer = new ConsumerBuilder<Dictionary<string, object>>(cfg).Build();
 ```
 
 </TabItem>
@@ -749,10 +743,12 @@ while(true){
 // 消费数据
 while (true)
 {
-    var consumerRes = consumer.Consume(100);
-    // process ConsumeResult
-    ProcessMsg(consumerRes);
-    consumer.Commit(consumerRes);
+    using (var result = consumer.Consume(500))
+    {
+        if (result == null) continue;
+        ProcessMsg(result);
+        consumer.Commit();
+    }
 }
 ```
 

docs/zh/08-connector/02-rest-api.mdx

@@ -257,6 +257,63 @@ curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timez
 - "INT UNSIGNED"
 - "BIGINT UNSIGNED"
 - "JSON"
+- "VARBINARY"
+- "GEOMETRY"
+
+`VARBINARY` 和 `GEOMETRY` 类型返回数据为 Hex 字符串，样例：
+
+准备数据
+
+```bash
+create database demo
+use demo
+create table t(ts timestamp,c1 varbinary(20),c2 geometry(100))
+insert into t values(now,'\x7f8290','point(100 100)')
+```
+
+执行查询
+
+```bash
+curl --location 'http://<fqdn>:<port>/rest/sql' \
+--header 'Content-Type: text/plain' \
+--header 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' \
+--data 'select * from demo.t'
+```
+
+返回结果
+
+```json
+{
+    "code": 0,
+    "column_meta": [
+        [
+            "ts",
+            "TIMESTAMP",
+            8
+        ],
+        [
+            "c1",
+            "VARBINARY",
+            20
+        ],
+        [
+            "c2",
+            "GEOMETRY",
+            100
+        ]
+    ],
+    "data": [
+        [
+            "2023-11-01T06:28:15.210Z",
+            "7f8290",
+            "010100000000000000000059400000000000005940"
+        ]
+    ],
+    "rows": 1
+}
+```
+
+- `010100000000000000000059400000000000005940` 为 `point(100 100)` 的 [Well-Known Binary (WKB)](https://libgeos.org/specifications/wkb/)  格式
 
 #### 错误
 

docs/zh/12-taos-sql/05-insert.md

@@ -159,6 +159,6 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c
 ```sql
 INSERT INTO meters(tbname, location, groupId, ts, current, phase) 
                 values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) 
-                values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
-                values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
+                ('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33)
+                ('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33)        
 ```

include/common/cos.h

@@ -34,7 +34,7 @@ extern int32_t tsS3UploadDelaySec;
 int32_t s3Init();
 void    s3CleanUp();
 int32_t s3PutObjectFromFile(const char *file, const char *object);
-int32_t s3PutObjectFromFile2(const char *file, const char *object);
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
 void    s3DeleteObjectsByPrefix(const char *prefix);
 void    s3DeleteObjects(const char *object_name[], int nobject);
 bool    s3Exists(const char *object_name);
@@ -43,6 +43,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
 int32_t s3GetObjectsByPrefix(const char *prefix, const char *path);
 void    s3EvictCache(const char *path, long object_size);
 long    s3Size(const char *object_name);
+int32_t s3GetObjectToFile(const char *object_name, char *fileName);
 
 #ifdef __cplusplus
 }

include/common/cos_cp.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_COMMON_COS_CP_H_
+#define _TD_COMMON_COS_CP_H_
+
+#include "os.h"
+#include "tdef.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum {
+  COS_CP_TYPE_UPLOAD,   // upload
+  COS_CP_TYPE_DOWNLOAD  // download
+} ECpType;
+
+typedef struct {
+  int32_t  index;      // the index of part, start from 0
+  int64_t  offset;     // the offset point of part
+  int64_t  size;       // the size of part
+  int      completed;  // COS_TRUE completed, COS_FALSE uncompleted
+  char     etag[128];  // the etag of part, for upload
+  uint64_t crc64;
+} SCheckpointPart;
+
+typedef struct {
+  ECpType   cp_type;  // 0 upload, 1 download
+  char      md5[64];  // the md5 of checkout content
+  TdFilePtr thefile;  // the handle of checkpoint file
+
+  char    file_path[TSDB_FILENAME_LEN];  // local file path
+  int64_t file_size;                     // local file size, for upload
+  int32_t file_last_modified;            // local file last modified time, for upload
+  char    file_md5[64];                  // md5 of the local file content, for upload, reserved
+
+  char    object_name[128];          // object name
+  int64_t object_size;               // object size, for download
+  char    object_last_modified[64];  // object last modified time, for download
+  char    object_etag[128];          // object etag, for download
+
+  char upload_id[128];  // upload id
+
+  int              part_num;   // the total number of parts
+  int64_t          part_size;  // the part size, byte
+  SCheckpointPart* parts;      // the parts of local or object, from 0
+} SCheckpoint;
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint);
+void    cos_cp_close(TdFilePtr fd);
+void    cos_cp_remove(char const* filepath);
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint);
+int32_t cos_cp_dump(SCheckpoint* checkpoint);
+void    cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes);
+void    cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64);
+void    cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                            char const* upload_id, int64_t part_size);
+bool    cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime);
+
+void cos_cp_build_download(SCheckpoint* checkpoint, char const* filepath, char const* object_name, int64_t object_size,
+                           char const* object_lmtime, char const* object_etag, int64_t part_size);
+bool cos_cp_is_valid_download(SCheckpoint* checkpoint, char const* object_name, int64_t object_size,
+                              char const* object_lmtime, char const* object_etag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_COMMON_COS_CP_H_*/

include/common/systable.h

@@ -50,6 +50,8 @@ extern "C" {
 #define TSDB_INS_TABLE_STREAM_TASKS      "ins_stream_tasks"
 #define TSDB_INS_TABLE_USER_PRIVILEGES   "ins_user_privileges"
 #define TSDB_INS_TABLE_VIEWS             "ins_views"
+#define TSDB_INS_TABLE_COMPACTS          "ins_compacts"
+#define TSDB_INS_TABLE_COMPACT_DETAILS   "ins_compact_details"
 
 #define TSDB_PERFORMANCE_SCHEMA_DB   "performance_schema"
 #define TSDB_PERFS_TABLE_SMAS        "perf_smas"

include/common/tcommon.h

@@ -253,13 +253,12 @@ typedef struct SQueryTableDataCond {
   STimeWindow  twindows;
   int64_t      startVersion;
   int64_t      endVersion;
+  bool         notLoadData;    // response the actual data, not only the rows in the attribute of info.row of ssdatablock
 } SQueryTableDataCond;
 
 int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
 void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
 
-int32_t tEncodeDataBlocks(void** buf, const SArray* blocks);
-void*   tDecodeDataBlocks(const void* buf, SArray** blocks);
 void    colDataDestroy(SColumnInfoData* pColData);
 
 //======================================================================================================================
@@ -294,7 +293,7 @@ typedef struct STableBlockDistInfo {
   int32_t  defMaxRows;
   int32_t  firstSeekTimeUs;
   uint32_t numOfInmemRows;
-  uint32_t numOfSmallBlocks;
+  uint32_t numOfSttRows;
   uint32_t numOfVgroups;
   int32_t  blockRowsHisto[20];
 } STableBlockDistInfo;

include/common/tglobal.h

@@ -75,6 +75,9 @@ extern int32_t tsElectInterval;
 extern int32_t tsHeartbeatInterval;
 extern int32_t tsHeartbeatTimeout;
 
+// vnode
+extern int64_t tsVndCommitMaxIntervalMs;
+
 // snode
 extern int32_t tsRsyncPort;
 extern char    tsCheckpointBackupDir[];
@@ -193,6 +196,7 @@ extern int64_t tsWalFsyncDataSizeLimit;
 
 // internal
 extern int32_t tsTransPullupInterval;
+extern int32_t tsCompactPullupInterval;
 extern int32_t tsMqRebalanceInterval;
 extern int32_t tsStreamCheckpointInterval;
 extern float   tsSinkDataRate;
@@ -211,6 +215,7 @@ extern int32_t tsMaxStreamBackendCache;
 extern int32_t tsPQSortMemThreshold;
 extern int32_t tsResolveFQDNRetryTime;
 
+extern bool tsExperimental;
 // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,

include/common/tmsg.h

@@ -65,8 +65,8 @@ typedef uint16_t tmsg_t;
 #define TMSG_INDEX(TYPE)    (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE))
 
 static inline bool tmsgIsValid(tmsg_t type) {
+  // static int8_t sz = sizeof(tMsgRangeDict) / sizeof(tMsgRangeDict[0]);
   int8_t maxSegIdx = TMSG_SEG_CODE(TDMT_MAX_MSG);
-
   int    segIdx = TMSG_SEG_CODE(type);
   if (segIdx >= 0 && segIdx < maxSegIdx) {
     return type < tMsgRangeDict[segIdx];
@@ -145,6 +145,8 @@ typedef enum _mgmt_table {
   TSDB_MGMT_TABLE_STREAM_TASKS,
   TSDB_MGMT_TABLE_PRIVILEGES,
   TSDB_MGMT_TABLE_VIEWS,
+  TSDB_MGMT_TABLE_COMPACT,
+  TSDB_MGMT_TABLE_COMPACT_DETAIL,
   TSDB_MGMT_TABLE_MAX,
 } EShowType;
 
@@ -296,7 +298,37 @@ typedef enum ENodeType {
   QUERY_NODE_SYNCDB_STMT,
   QUERY_NODE_GRANT_STMT,
   QUERY_NODE_REVOKE_STMT,
-  QUERY_NODE_SHOW_DNODES_STMT,
+  // placeholder for [152, 180]
+  QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
+  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
+  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
+  QUERY_NODE_KILL_CONNECTION_STMT,
+  QUERY_NODE_KILL_QUERY_STMT,
+  QUERY_NODE_KILL_TRANSACTION_STMT,
+  QUERY_NODE_KILL_COMPACT_STMT,
+  QUERY_NODE_DELETE_STMT,
+  QUERY_NODE_INSERT_STMT,
+  QUERY_NODE_QUERY,
+  QUERY_NODE_SHOW_DB_ALIVE_STMT,
+  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_RESTORE_DNODE_STMT,
+  QUERY_NODE_RESTORE_QNODE_STMT,
+  QUERY_NODE_RESTORE_MNODE_STMT,
+  QUERY_NODE_RESTORE_VNODE_STMT,
+  QUERY_NODE_PAUSE_STREAM_STMT,
+  QUERY_NODE_RESUME_STREAM_STMT,
+  QUERY_NODE_CREATE_VIEW_STMT,
+  QUERY_NODE_DROP_VIEW_STMT,
+
+  // show statement nodes
+  // see 'sysTableShowAdapter', 'SYSTABLE_SHOW_TYPE_OFFSET'
+  QUERY_NODE_SHOW_DNODES_STMT = 400,
   QUERY_NODE_SHOW_MNODES_STMT,
   QUERY_NODE_SHOW_MODULES_STMT,
   QUERY_NODE_SHOW_QNODES_STMT,
@@ -325,31 +357,8 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_VNODES_STMT,
   QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
   QUERY_NODE_SHOW_VIEWS_STMT,
-  QUERY_NODE_SHOW_CREATE_VIEW_STMT,
-  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
-  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
-  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
-  QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT,
-  QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT,
-  QUERY_NODE_SHOW_SCORES_STMT,
-  QUERY_NODE_SHOW_TABLE_TAGS_STMT,
-  QUERY_NODE_KILL_CONNECTION_STMT,
-  QUERY_NODE_KILL_QUERY_STMT,
-  QUERY_NODE_KILL_TRANSACTION_STMT,
-  QUERY_NODE_DELETE_STMT,
-  QUERY_NODE_INSERT_STMT,
-  QUERY_NODE_QUERY,
-  QUERY_NODE_SHOW_DB_ALIVE_STMT,
-  QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
-  QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
-  QUERY_NODE_RESTORE_DNODE_STMT,
-  QUERY_NODE_RESTORE_QNODE_STMT,
-  QUERY_NODE_RESTORE_MNODE_STMT,
-  QUERY_NODE_RESTORE_VNODE_STMT,
-  QUERY_NODE_PAUSE_STREAM_STMT,
-  QUERY_NODE_RESUME_STREAM_STMT,
-  QUERY_NODE_CREATE_VIEW_STMT,
-  QUERY_NODE_DROP_VIEW_STMT,
+  QUERY_NODE_SHOW_COMPACTS_STMT,
+  QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
 
   // logic plan node
   QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@@ -1381,6 +1390,24 @@ int32_t tSerializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
 int32_t tDeserializeSCompactDbReq(void* buf, int32_t bufLen, SCompactDbReq* pReq);
 void    tFreeSCompactDbReq(SCompactDbReq* pReq);
 
+typedef struct {
+  int32_t compactId;
+  int8_t bAccepted;
+} SCompactDbRsp;
+
+int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
+int32_t tDeserializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
+
+typedef struct {
+  int32_t compactId;
+  int32_t sqlLen;
+  char*   sql;
+} SKillCompactReq;
+
+int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
+int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
+void    tFreeSKillCompactReq(SKillCompactReq *pReq);
+
 typedef struct {
   char    name[TSDB_FUNC_NAME_LEN];
   int8_t  igExists;
@@ -1659,6 +1686,26 @@ int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pR
 int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq);
 int32_t tFreeSCreateVnodeReq(SCreateVnodeReq* pReq);
 
+typedef struct {
+  int32_t compactId;
+  int32_t vgId;
+  int32_t dnodeId;
+} SQueryCompactProgressReq;
+
+int32_t tSerializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
+int32_t tDeserializeSQueryCompactProgressReq(void* buf, int32_t bufLen, SQueryCompactProgressReq* pReq);
+
+typedef struct {
+  int32_t compactId;
+  int32_t vgId;
+  int32_t dnodeId;
+  int32_t numberFileset;
+  int32_t finished;
+} SQueryCompactProgressRsp;
+
+int32_t tSerializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
+int32_t tDeserializeSQueryCompactProgressRsp(void* buf, int32_t bufLen, SQueryCompactProgressRsp* pReq);
+
 typedef struct {
   int32_t vgId;
   int32_t dnodeId;
@@ -1686,11 +1733,21 @@ typedef struct {
   char        db[TSDB_DB_FNAME_LEN];
   int64_t     compactStartTime;
   STimeWindow tw;
+  int32_t     compactId;
 } SCompactVnodeReq;
 
 int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
 int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
 
+typedef struct {
+  int32_t     compactId;
+  int32_t     vgId;
+  int32_t     dnodeId;
+} SVKillCompactReq;
+
+int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
+int32_t tDeserializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
+
 typedef struct {
   int32_t vgVersion;
   int32_t buffer;
@@ -1886,8 +1943,9 @@ typedef struct {
   char    db[TSDB_DB_FNAME_LEN];
   char    tb[TSDB_TABLE_NAME_LEN];
   char    user[TSDB_USER_LEN];
-  char    filterTb[TSDB_TABLE_NAME_LEN];
+  char    filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
   int64_t showId;
+  int64_t compactId; // for compact
 } SRetrieveTableReq;
 
 typedef struct SSysTableSchema {

include/common/tmsgdef.h

@@ -61,7 +61,7 @@
   #undef TD_CLOSE_MSG_SEG
   #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM,
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP)
-  #define TD_CLOSE_MSG_SEG(TYPE)
+  #define TD_CLOSE_MSG_SEG(type)
 
   int32_t tMsgDict[] = {
 
@@ -85,10 +85,9 @@
   #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) TYPE, TYPE##_RSP,
   #define TD_CLOSE_MSG_SEG(TYPE) TYPE,
 
-
   enum { // WARN: new msg should be appended to segment tail
 #endif
-  TD_NEW_MSG_SEG(TDMT_DND_MSG)
+  TD_NEW_MSG_SEG(TDMT_DND_MSG)    // 0<<8
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_MNODE, "dnode-create-mnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_DROP_MNODE, "dnode-drop-mnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_DND_CREATE_QNODE, "dnode-create-qnode", NULL, NULL)
@@ -110,8 +109,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_DND_MSG)
 
-
-  TD_NEW_MSG_SEG(TDMT_MND_MSG)
+  TD_NEW_MSG_SEG(TDMT_MND_MSG)  // 1<<8
   TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_ACCT, "create-acct", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_ACCT, "alter-acct", NULL, NULL)
@@ -210,16 +208,19 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_MSG)  // 2<<8
   TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TABLE, "create-table", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TABLE, "alter-table", NULL, NULL)
@@ -266,10 +267,12 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_VND_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_SCH_MSG)
+  TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY, "query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_MERGE_QUERY, "merge-query", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SCH_QUERY_CONTINUE, "query-continue", NULL, NULL)
@@ -285,13 +288,13 @@ enum { // WARN: new msg should be appended to segment tail
   TD_CLOSE_MSG_SEG(TDMT_END_SCH_MSG)
 
 
-  TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
-  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
+  TD_NEW_MSG_SEG(TDMT_STREAM_MSG)   //4 << 8
+  TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)  //1025 1026
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_UNUSED1, "stream-unused1", NULL, NULL)
-  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL)  //1035 1036
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_CHECKPOINT_READY, "stream-checkpoint-ready", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_REPORT_CHECKPOINT, "stream-report-checkpoint", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESTORE_CHECKPOINT, "stream-restore-checkpoint", NULL, NULL)
@@ -302,12 +305,11 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_MON_MSG)
+  TD_NEW_MSG_SEG(TDMT_MON_MSG)  //5 << 8
   TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_MON_MSG)
 
-
-  TD_NEW_MSG_SEG(TDMT_SYNC_MSG)
+  TD_NEW_MSG_SEG(TDMT_SYNC_MSG) //6 << 8
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT, "sync-timer", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_TIMEOUT_ELECTION, "sync-elect", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_SYNC_PING_REPLY, "sync-ping-reply", NULL, NULL) // no longer used
@@ -339,7 +341,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY, "vnode-stream-scan-history", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_SCAN_HISTORY_FINISH, "vnode-stream-scan-history-finish", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_CHECK_POINT_SOURCE, "vnode-stream-checkpoint-source", NULL, NULL)
@@ -349,7 +351,7 @@ enum { // WARN: new msg should be appended to segment tail
   TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
 
-  TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)
+  TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG)  //8 << 8
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp)
   TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset)
@@ -365,6 +367,7 @@ enum { // WARN: new msg should be appended to segment tail
 
   TD_NEW_MSG_SEG(TDMT_MAX_MSG)  // msg end mark
   TD_CLOSE_MSG_SEG(TDMT_END_MAX_MSG)
+
   #if defined(TD_MSG_NUMBER_)
     TDMT_MAX
   #endif

include/common/ttokendef.h

@@ -189,184 +189,180 @@
 #define TK_ALIVE                          170
 #define TK_VIEWS                          171
 #define TK_VIEW                           172
-#define TK_NORMAL                         173
-#define TK_CHILD                          174
-#define TK_LIKE                           175
-#define TK_TBNAME                         176
-#define TK_QTAGS                          177
-#define TK_AS                             178
-#define TK_SYSTEM                         179
-#define TK_INDEX                          180
-#define TK_FUNCTION                       181
-#define TK_INTERVAL                       182
-#define TK_COUNT                          183
-#define TK_LAST_ROW                       184
-#define TK_META                           185
-#define TK_ONLY                           186
-#define TK_TOPIC                          187
-#define TK_CONSUMER                       188
-#define TK_GROUP                          189
-#define TK_DESC                           190
-#define TK_DESCRIBE                       191
-#define TK_RESET                          192
-#define TK_QUERY                          193
-#define TK_CACHE                          194
-#define TK_EXPLAIN                        195
-#define TK_ANALYZE                        196
-#define TK_VERBOSE                        197
-#define TK_NK_BOOL                        198
-#define TK_RATIO                          199
-#define TK_NK_FLOAT                       200
-#define TK_OUTPUTTYPE                     201
-#define TK_AGGREGATE                      202
-#define TK_BUFSIZE                        203
-#define TK_LANGUAGE                       204
-#define TK_REPLACE                        205
-#define TK_STREAM                         206
-#define TK_INTO                           207
-#define TK_PAUSE                          208
-#define TK_RESUME                         209
-#define TK_TRIGGER                        210
-#define TK_AT_ONCE                        211
-#define TK_WINDOW_CLOSE                   212
-#define TK_IGNORE                         213
-#define TK_EXPIRED                        214
-#define TK_FILL_HISTORY                   215
-#define TK_UPDATE                         216
-#define TK_SUBTABLE                       217
-#define TK_UNTREATED                      218
-#define TK_KILL                           219
-#define TK_CONNECTION                     220
-#define TK_TRANSACTION                    221
-#define TK_BALANCE                        222
-#define TK_VGROUP                         223
-#define TK_LEADER                         224
-#define TK_MERGE                          225
-#define TK_REDISTRIBUTE                   226
-#define TK_SPLIT                          227
-#define TK_DELETE                         228
-#define TK_INSERT                         229
-#define TK_NULL                           230
-#define TK_NK_QUESTION                    231
-#define TK_NK_ALIAS                       232
-#define TK_NK_ARROW                       233
-#define TK_ROWTS                          234
-#define TK_QSTART                         235
-#define TK_QEND                           236
-#define TK_QDURATION                      237
-#define TK_WSTART                         238
-#define TK_WEND                           239
-#define TK_WDURATION                      240
-#define TK_IROWTS                         241
-#define TK_ISFILLED                       242
-#define TK_CAST                           243
-#define TK_NOW                            244
-#define TK_TODAY                          245
-#define TK_TIMEZONE                       246
-#define TK_CLIENT_VERSION                 247
-#define TK_SERVER_VERSION                 248
-#define TK_SERVER_STATUS                  249
-#define TK_CURRENT_USER                   250
-#define TK_CASE                           251
-#define TK_WHEN                           252
-#define TK_THEN                           253
-#define TK_ELSE                           254
-#define TK_BETWEEN                        255
-#define TK_IS                             256
-#define TK_NK_LT                          257
-#define TK_NK_GT                          258
-#define TK_NK_LE                          259
-#define TK_NK_GE                          260
-#define TK_NK_NE                          261
-#define TK_MATCH                          262
-#define TK_NMATCH                         263
-#define TK_CONTAINS                       264
-#define TK_IN                             265
-#define TK_JOIN                           266
-#define TK_INNER                          267
-#define TK_SELECT                         268
-#define TK_NK_HINT                        269
-#define TK_DISTINCT                       270
-#define TK_WHERE                          271
-#define TK_PARTITION                      272
-#define TK_BY                             273
-#define TK_SESSION                        274
-#define TK_STATE_WINDOW                   275
-#define TK_EVENT_WINDOW                   276
-#define TK_SLIDING                        277
-#define TK_FILL                           278
-#define TK_VALUE                          279
-#define TK_VALUE_F                        280
-#define TK_NONE                           281
-#define TK_PREV                           282
-#define TK_NULL_F                         283
-#define TK_LINEAR                         284
-#define TK_NEXT                           285
-#define TK_HAVING                         286
-#define TK_RANGE                          287
-#define TK_EVERY                          288
-#define TK_ORDER                          289
-#define TK_SLIMIT                         290
-#define TK_SOFFSET                        291
-#define TK_LIMIT                          292
-#define TK_OFFSET                         293
-#define TK_ASC                            294
-#define TK_NULLS                          295
-#define TK_ABORT                          296
-#define TK_AFTER                          297
-#define TK_ATTACH                         298
-#define TK_BEFORE                         299
-#define TK_BEGIN                          300
-#define TK_BITAND                         301
-#define TK_BITNOT                         302
-#define TK_BITOR                          303
-#define TK_BLOCKS                         304
-#define TK_CHANGE                         305
-#define TK_COMMA                          306
-#define TK_CONCAT                         307
-#define TK_CONFLICT                       308
-#define TK_COPY                           309
-#define TK_DEFERRED                       310
-#define TK_DELIMITERS                     311
-#define TK_DETACH                         312
-#define TK_DIVIDE                         313
-#define TK_DOT                            314
-#define TK_EACH                           315
-#define TK_FAIL                           316
-#define TK_FILE                           317
-#define TK_FOR                            318
-#define TK_GLOB                           319
-#define TK_ID                             320
-#define TK_IMMEDIATE                      321
-#define TK_IMPORT                         322
-#define TK_INITIALLY                      323
-#define TK_INSTEAD                        324
-#define TK_ISNULL                         325
-#define TK_KEY                            326
-#define TK_MODULES                        327
-#define TK_NK_BITNOT                      328
-#define TK_NK_SEMI                        329
-#define TK_NOTNULL                        330
-#define TK_OF                             331
-#define TK_PLUS                           332
-#define TK_PRIVILEGE                      333
-#define TK_RAISE                          334
-#define TK_RESTRICT                       335
-#define TK_ROW                            336
-#define TK_SEMI                           337
-#define TK_STAR                           338
-#define TK_STATEMENT                      339
-#define TK_STRICT                         340
-#define TK_STRING                         341
-#define TK_TIMES                          342
-#define TK_VALUES                         343
-#define TK_VARIABLE                       344
-#define TK_WAL                            345
-
-
-
-
-
+#define TK_COMPACTS                       173
+#define TK_NORMAL                         174
+#define TK_CHILD                          175
+#define TK_LIKE                           176
+#define TK_TBNAME                         177
+#define TK_QTAGS                          178
+#define TK_AS                             179
+#define TK_SYSTEM                         180
+#define TK_INDEX                          181
+#define TK_FUNCTION                       182
+#define TK_INTERVAL                       183
+#define TK_COUNT                          184
+#define TK_LAST_ROW                       185
+#define TK_META                           186
+#define TK_ONLY                           187
+#define TK_TOPIC                          188
+#define TK_CONSUMER                       189
+#define TK_GROUP                          190
+#define TK_DESC                           191
+#define TK_DESCRIBE                       192
+#define TK_RESET                          193
+#define TK_QUERY                          194
+#define TK_CACHE                          195
+#define TK_EXPLAIN                        196
+#define TK_ANALYZE                        197
+#define TK_VERBOSE                        198
+#define TK_NK_BOOL                        199
+#define TK_RATIO                          200
+#define TK_NK_FLOAT                       201
+#define TK_OUTPUTTYPE                     202
+#define TK_AGGREGATE                      203
+#define TK_BUFSIZE                        204
+#define TK_LANGUAGE                       205
+#define TK_REPLACE                        206
+#define TK_STREAM                         207
+#define TK_INTO                           208
+#define TK_PAUSE                          209
+#define TK_RESUME                         210
+#define TK_TRIGGER                        211
+#define TK_AT_ONCE                        212
+#define TK_WINDOW_CLOSE                   213
+#define TK_IGNORE                         214
+#define TK_EXPIRED                        215
+#define TK_FILL_HISTORY                   216
+#define TK_UPDATE                         217
+#define TK_SUBTABLE                       218
+#define TK_UNTREATED                      219
+#define TK_KILL                           220
+#define TK_CONNECTION                     221
+#define TK_TRANSACTION                    222
+#define TK_BALANCE                        223
+#define TK_VGROUP                         224
+#define TK_LEADER                         225
+#define TK_MERGE                          226
+#define TK_REDISTRIBUTE                   227
+#define TK_SPLIT                          228
+#define TK_DELETE                         229
+#define TK_INSERT                         230
+#define TK_NULL                           231
+#define TK_NK_QUESTION                    232
+#define TK_NK_ALIAS                       233
+#define TK_NK_ARROW                       234
+#define TK_ROWTS                          235
+#define TK_QSTART                         236
+#define TK_QEND                           237
+#define TK_QDURATION                      238
+#define TK_WSTART                         239
+#define TK_WEND                           240
+#define TK_WDURATION                      241
+#define TK_IROWTS                         242
+#define TK_ISFILLED                       243
+#define TK_CAST                           244
+#define TK_NOW                            245
+#define TK_TODAY                          246
+#define TK_TIMEZONE                       247
+#define TK_CLIENT_VERSION                 248
+#define TK_SERVER_VERSION                 249
+#define TK_SERVER_STATUS                  250
+#define TK_CURRENT_USER                   251
+#define TK_CASE                           252
+#define TK_WHEN                           253
+#define TK_THEN                           254
+#define TK_ELSE                           255
+#define TK_BETWEEN                        256
+#define TK_IS                             257
+#define TK_NK_LT                          258
+#define TK_NK_GT                          259
+#define TK_NK_LE                          260
+#define TK_NK_GE                          261
+#define TK_NK_NE                          262
+#define TK_MATCH                          263
+#define TK_NMATCH                         264
+#define TK_CONTAINS                       265
+#define TK_IN                             266
+#define TK_JOIN                           267
+#define TK_INNER                          268
+#define TK_SELECT                         269
+#define TK_NK_HINT                        270
+#define TK_DISTINCT                       271
+#define TK_WHERE                          272
+#define TK_PARTITION                      273
+#define TK_BY                             274
+#define TK_SESSION                        275
+#define TK_STATE_WINDOW                   276
+#define TK_EVENT_WINDOW                   277
+#define TK_SLIDING                        278
+#define TK_FILL                           279
+#define TK_VALUE                          280
+#define TK_VALUE_F                        281
+#define TK_NONE                           282
+#define TK_PREV                           283
+#define TK_NULL_F                         284
+#define TK_LINEAR                         285
+#define TK_NEXT                           286
+#define TK_HAVING                         287
+#define TK_RANGE                          288
+#define TK_EVERY                          289
+#define TK_ORDER                          290
+#define TK_SLIMIT                         291
+#define TK_SOFFSET                        292
+#define TK_LIMIT                          293
+#define TK_OFFSET                         294
+#define TK_ASC                            295
+#define TK_NULLS                          296
+#define TK_ABORT                          297
+#define TK_AFTER                          298
+#define TK_ATTACH                         299
+#define TK_BEFORE                         300
+#define TK_BEGIN                          301
+#define TK_BITAND                         302
+#define TK_BITNOT                         303
+#define TK_BITOR                          304
+#define TK_BLOCKS                         305
+#define TK_CHANGE                         306
+#define TK_COMMA                          307
+#define TK_CONCAT                         308
+#define TK_CONFLICT                       309
+#define TK_COPY                           310
+#define TK_DEFERRED                       311
+#define TK_DELIMITERS                     312
+#define TK_DETACH                         313
+#define TK_DIVIDE                         314
+#define TK_DOT                            315
+#define TK_EACH                           316
+#define TK_FAIL                           317
+#define TK_FILE                           318
+#define TK_FOR                            319
+#define TK_GLOB                           320
+#define TK_ID                             321
+#define TK_IMMEDIATE                      322
+#define TK_IMPORT                         323
+#define TK_INITIALLY                      324
+#define TK_INSTEAD                        325
+#define TK_ISNULL                         326
+#define TK_KEY                            327
+#define TK_MODULES                        328
+#define TK_NK_BITNOT                      329
+#define TK_NK_SEMI                        330
+#define TK_NOTNULL                        331
+#define TK_OF                             332
+#define TK_PLUS                           333
+#define TK_PRIVILEGE                      334
+#define TK_RAISE                          335
+#define TK_RESTRICT                       336
+#define TK_ROW                            337
+#define TK_SEMI                           338
+#define TK_STAR                           339
+#define TK_STATEMENT                      340
+#define TK_STRICT                         341
+#define TK_STRING                         342
+#define TK_TIMES                          343
+#define TK_VALUES                         344
+#define TK_VARIABLE                       345
+#define TK_WAL                            346
 
 #define TK_NK_SPACE         600
 #define TK_NK_COMMENT       601

include/common/ttypes.h

@@ -275,9 +275,11 @@ typedef struct {
 #define IS_VALID_TINYINT(_t)   ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
 #define IS_VALID_SMALLINT(_t)  ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
 #define IS_VALID_INT(_t)       ((_t) >= INT32_MIN && (_t) <= INT32_MAX)
+#define IS_VALID_INT64(_t)     ((_t) >= INT64_MIN && (_t) <= INT64_MAX)
 #define IS_VALID_UTINYINT(_t)  ((_t) >= 0 && (_t) <= UINT8_MAX)
 #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX)
 #define IS_VALID_UINT(_t)      ((_t) >= 0 && (_t) <= UINT32_MAX)
+#define IS_VALID_UINT64(_t)    ((_t) >= 0 && (_t) <= UINT64_MAX)
 #define IS_VALID_FLOAT(_t)     ((_t) >= -FLT_MAX && (_t) <= FLT_MAX)
 #define IS_VALID_DOUBLE(_t)    ((_t) >= -DBL_MAX && (_t) <= DBL_MAX)
 

include/common/tvariant.h

@@ -37,6 +37,10 @@ typedef struct SVariant {
   };
 } SVariant;
 
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value);
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value);
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double *value);
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value);
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value);
 

include/dnode/snode/snode.h

@@ -45,6 +45,7 @@ typedef struct {
  */
 SSnode *sndOpen(const char *path, const SSnodeOpt *pOption);
 
+int32_t sndInit(SSnode * pSnode);
 /**
  * @brief Stop Snode in Dnode.
  *

include/dnode/vnode/tqCommon.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TQ_COMMON_H
+#define TDENGINE_TQ_COMMON_H
+
+// message process
+int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart);
+int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored);
+int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessScanHistoryFinishReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessScanHistoryFinishRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
+int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
+int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored);
+int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
+int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader);
+int32_t startStreamTasks(SStreamMeta* pMeta);
+int32_t resetStreamTaskStatus(SStreamMeta* pMeta);
+
+#endif  // TDENGINE_TQ_COMMON_H

include/libs/executor/storageapi.h

@@ -152,10 +152,22 @@ typedef struct {
 
 // clang-format off
 /*-------------------------------------------------new api format---------------------------------------------------*/
+typedef enum {
+  TSD_READER_NOTIFY_DURATION_START,
+  TSD_READER_NOTIFY_NEXT_DURATION_BLOCK,
+} ETsdReaderNotifyType;
+
+typedef union {
+  struct {
+    int32_t filesetId;
+  } duration;
+} STsdReaderNotifyInfo;
+
+typedef void (*TsdReaderNotifyCbFn)(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param);
+
 typedef struct TsdReader {
   int32_t      (*tsdReaderOpen)(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
-                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, bool countOnly,
-                           SHashObj** pIgnoreTables);
+                           SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables);
   void         (*tsdReaderClose)();
   void         (*tsdSetReaderTaskId)(void *pReader, const char *pId);
   int32_t      (*tsdSetQueryTableList)();
@@ -170,6 +182,9 @@ typedef struct TsdReader {
   int32_t      (*tsdReaderGetDataBlockDistInfo)();
   int64_t      (*tsdReaderGetNumOfInMemRows)();
   void         (*tsdReaderNotifyClosing)();
+
+  void         (*tsdSetFilesetDelimited)(void* pReader);
+  void         (*tsdSetSetNotifyCb)(void* pReader, TsdReaderNotifyCbFn notifyFn, void* param);
 } TsdReader;
 
 typedef struct SStoreCacheReader {
@@ -255,7 +270,8 @@ typedef struct SStoreMeta {
 
   int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid);  // tsdbGetTableSchema
   int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
-  void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables, int64_t* numOfNormalTables);
+  void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
+                       int64_t* numOfNormalTables);
 
   int64_t (*getNumOfRowsInMem)(void* pVnode);
 
@@ -351,6 +367,8 @@ typedef struct SStateStore {
   TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol);
   bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
   bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid);
+  bool (*isIncrementalTimeStamp)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts);
+
   void (*updateInfoDestroy)(SUpdateInfo* pInfo);
   void (*windowSBfDelete)(SUpdateInfo* pInfo, uint64_t count);
   void (*windowSBfAdd)(SUpdateInfo* pInfo, uint64_t count);
@@ -380,6 +398,7 @@ typedef struct SStateStore {
   void (*streamStateDestroy)(SStreamState* pState, bool remove);
   int32_t (*streamStateDeleteCheckPoint)(SStreamState* pState, TSKEY mark);
   void (*streamStateReloadInfo)(SStreamState* pState, TSKEY ts);
+  void (*streamStateCopyBackend)(SStreamState* src, SStreamState* dst);
 } SStateStore;
 
 typedef struct SStorageAPI {

include/libs/function/function.h

@@ -171,6 +171,7 @@ typedef struct {
   int32_t                  taskId;
   int64_t                  streamId;
   int64_t                  streamBackendRid;
+  int8_t                   dump;
 } SStreamState;
 
 typedef struct SFunctionStateStore {

include/libs/nodes/cmdnodes.h

@@ -46,6 +46,10 @@ extern "C" {
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
 #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
 
+#define COMPACT_DB_RESULT_COLS 3
+#define COMPACT_DB_RESULT_FIELD1_LEN 32
+#define COMPACT_DB_RESULT_FIELD3_LEN 128
+
 #define SHOW_ALIVE_RESULT_COLS 1
 
 #define BIT_FLAG_MASK(n)              (1 << n)
@@ -335,6 +339,15 @@ typedef struct SShowTableTagsStmt {
   SNodeList* pTags;
 } SShowTableTagsStmt;
 
+typedef struct SShowCompactsStmt {
+  ENodeType type;
+} SShowCompactsStmt;
+
+typedef struct SShowCompactDetailsStmt {
+  ENodeType type;
+  SNode* pCompactId;
+} SShowCompactDetailsStmt;
+
 typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
 
 typedef struct SIndexOptions {

include/libs/nodes/plannodes.h

@@ -118,6 +118,7 @@ typedef struct SScanLogicNode {
   bool          igLastNull;
   bool          groupOrderScan;
   bool          onlyMetaCtbIdx; // for tag scan with no tbname
+  bool          filesetDelimited; // returned blocks delimited by fileset
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -432,6 +433,7 @@ typedef struct STableScanPhysiNode {
   int8_t         igExpired;
   bool           assignBlockUid;
   int8_t         igCheckUpdate;
+  bool           filesetDelimited;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;

include/libs/nodes/querynodes.h

@@ -449,6 +449,7 @@ typedef struct SVnodeModifyOpStmt {
   SHashObj*             pSubTableHashObj;    // SHashObj<table_name, STableMeta*>
   SHashObj*             pTableNameHashObj;   // set of table names for refreshing meta, sync mode
   SHashObj*             pDbFNameHashObj;     // set of db names for refreshing meta, sync mode
+  SHashObj*             pTableCxtHashObj;    // temp SHashObj<tuid, STableDataCxt*> for single request
   SArray*               pVgDataBlocks;  // SArray<SVgroupDataCxt*>
   SVCreateTbReq*        pCreateTblReq;
   TdFilePtr             fp;

include/libs/stream/streamState.h

@@ -35,6 +35,7 @@ int32_t       streamStateBegin(SStreamState* pState);
 int32_t       streamStateCommit(SStreamState* pState);
 void          streamStateDestroy(SStreamState* pState, bool remove);
 int32_t       streamStateDeleteCheckPoint(SStreamState* pState, TSKEY mark);
+int32_t       streamStateDelTaskDb(SStreamState* pState);
 
 int32_t streamStateFuncPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen);
 int32_t streamStateFuncGet(SStreamState* pState, const SWinKey* key, void** ppVal, int32_t* pVLen);
@@ -95,6 +96,9 @@ int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char*
 int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal);
 
 void streamStateReloadInfo(SStreamState* pState, TSKEY ts);
+
+void streamStateCopyBackend(SStreamState* src, SStreamState* dst);
+
 SStreamStateCur* createStreamStateCursor();
 
 /***compare func **/

include/libs/stream/tstream.h

@@ -34,7 +34,6 @@ extern "C" {
 #define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F)
 #define SIZE_IN_KiB(_v) ((_v) / ONE_KiB_F)
 
-
 #define TASK_DOWNSTREAM_READY      0x0
 #define TASK_DOWNSTREAM_NOT_READY  0x1
 #define TASK_DOWNSTREAM_NOT_LEADER 0x2
@@ -51,11 +50,17 @@ extern "C" {
     (_t)->hTaskInfo.id.streamId = 0;       \
   } while (0)
 
+#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1)
+#define STREAM_EXEC_START_ALL_TASKS_ID     (-2)
+#define STREAM_EXEC_RESTART_ALL_TASKS_ID   (-3)
+
 typedef struct SStreamTask   SStreamTask;
 typedef struct SStreamQueue  SStreamQueue;
 typedef struct SStreamTaskSM SStreamTaskSM;
 
 #define SSTREAM_TASK_VER              2
+#define SSTREAM_TASK_INCOMPATIBLE_VER 1
+#define SSTREAM_TASK_NEED_CONVERT_VER 2
 
 enum {
   STREAM_STATUS__NORMAL = 0,
@@ -107,6 +112,7 @@ typedef enum {
   TASK_LEVEL__SOURCE = 1,
   TASK_LEVEL__AGG,
   TASK_LEVEL__SINK,
+  TASK_LEVEL_SMA,
 } ETASK_LEVEL;
 
 enum {
@@ -210,9 +216,6 @@ int32_t         streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem);
 SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue);
 #endif
 
-int32_t streamInit();
-void    streamCleanUp();
-
 SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type);
 void               streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit);
 
@@ -301,11 +304,16 @@ typedef struct SStreamTaskId {
 typedef struct SCheckpointInfo {
   int64_t startTs;
   int64_t checkpointId;
+
   int64_t checkpointVer;  // latest checkpointId version
-  int64_t processedVer;        // already processed ver, that has generated results version.
+  int64_t processedVer;
   int64_t nextProcessVer;  // current offset in WAL, not serialize it
   int64_t failedId;        // record the latest failed checkpoint id
+  int64_t checkpointingId;
+  int32_t downstreamAlignNum;
+  int32_t checkpointNotReadyTasks;
   bool    dispatchCheckpointTrigger;
+  int64_t msgVer;
 } SCheckpointInfo;
 
 typedef struct SStreamStatus {
@@ -444,12 +452,11 @@ struct SStreamTask {
   int64_t             checkReqId;
   SArray*             checkReqIds;  // shuffle
   int32_t             refCnt;
-  int64_t             checkpointingId;
-  int32_t             checkpointAlignCnt;
-  int32_t             checkpointNotReadyTasks;
   int32_t             transferStateAlignCnt;
   struct SStreamMeta* pMeta;
   SSHashObj*          pNameMap;
+  void*               pBackend;
+  int64_t             backendRefId;
   char                reserve[256];
 };
 
@@ -487,20 +494,22 @@ typedef struct SStreamMeta {
   int32_t         walScanCounter;
   void*           streamBackend;
   int64_t         streamBackendRid;
-  SHashObj*       pTaskBackendUnique;
+  SHashObj*       pTaskDbUnique;
   TdThreadMutex   backendMutex;
   SMetaHbInfo*    pHbInfo;
   STaskUpdateInfo updateInfo;
+  SHashObj*       pUpdateTaskSet;
   int32_t         numOfStreamTasks;  // this value should be increased when a new task is added into the meta
   int32_t         numOfPausedTasks;
-  int32_t         chkptNotReadyTasks;
   int64_t         rid;
 
   int64_t  chkpId;
+  int32_t  chkpCap;
   SArray*  chkpSaved;
   SArray*  chkpInUse;
-  int32_t  chkpCap;
   SRWLatch chkpDirLock;
+  void*    qHandle;
+  void*    bkdChkptMgt;
 } SStreamMeta;
 
 int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
@@ -591,7 +600,7 @@ typedef struct {
   int32_t downstreamNodeId;
   int32_t downstreamTaskId;
   int32_t childId;
-  int32_t oldStage;
+  int64_t oldStage;
   int8_t  status;
 } SStreamTaskCheckRsp;
 
@@ -764,7 +773,8 @@ void    initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen)
 // recover and fill history
 void streamTaskCheckDownstream(SStreamTask* pTask);
 
-int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
+int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage,
+                              int64_t* oldStage);
 int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
 void    streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
 bool    streamTaskAllUpstreamClosed(SStreamTask* pTask);
@@ -827,14 +837,14 @@ int32_t      streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int3
 int32_t      streamMetaGetNumOfTasks(SStreamMeta* pMeta);
 SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
 void         streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
-int32_t      streamMetaReopen(SStreamMeta* pMeta);
+void         streamMetaClear(SStreamMeta* pMeta);
 void         streamMetaInitBackend(SStreamMeta* pMeta);
 int32_t      streamMetaCommit(SStreamMeta* pMeta);
 int32_t      streamMetaLoadAllTasks(SStreamMeta* pMeta);
 int64_t      streamMetaGetLatestCheckpointId(SStreamMeta* pMeta);
 void         streamMetaNotifyClose(SStreamMeta* pMeta);
+int32_t      streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);
 void         streamMetaStartHb(SStreamMeta* pMeta);
-void         streamMetaInitForSnode(SStreamMeta* pMeta);
 bool         streamMetaTaskInTimer(SStreamMeta* pMeta);
 int32_t      streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs,
                                                   int64_t endTs, bool ready);
@@ -848,7 +858,7 @@ void         streamMetaResetStartInfo(STaskStartInfo* pMeta);
 int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
 int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
 int32_t streamTaskBuildCheckpoint(SStreamTask* pTask);
-void    streamTaskClearCheckInfo(SStreamTask* pTask);
+void    streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg);
 int32_t streamAlignTransferState(SStreamTask* pTask);
 int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
 int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,
@@ -856,6 +866,8 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
 int32_t buildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
                                  int8_t isSucceed);
 
+SStreamTaskSM* streamCreateStateMachine(SStreamTask* pTask);
+void*          streamDestroyStateMachine(SStreamTaskSM* pSM);
 #ifdef __cplusplus
 }
 #endif

include/libs/stream/tstreamUpdate.h

@@ -55,6 +55,7 @@ int32_t      updateInfoSerialize(void *buf, int32_t bufLen, const SUpdateInfo *p
 int32_t      updateInfoDeserialize(void *buf, int32_t bufLen, SUpdateInfo *pInfo);
 void         windowSBfDelete(SUpdateInfo *pInfo, uint64_t count);
 void         windowSBfAdd(SUpdateInfo *pInfo, uint64_t count);
+bool         isIncrementalTimeStamp(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts);
 
 #ifdef __cplusplus
 }

include/libs/sync/sync.h

@@ -33,11 +33,12 @@ extern "C" {
 #define SYNC_MAX_PROGRESS_WAIT_MS    4000
 #define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20)
 #define SYNC_MAX_RECV_TIME_RANGE_MS  1200
-#define SYNC_DEL_WAL_MS              (1000 * 60)
 #define SYNC_ADD_QUORUM_COUNT        3
 #define SYNC_VNODE_LOG_RETENTION     (TSDB_SYNC_LOG_BUFFER_RETENTION + 1)
 #define SNAPSHOT_WAIT_MS             1000 * 5
 
+#define SYNC_WAL_LOG_RETENTION_SIZE (8LL * 1024 * 1024 * 1024)
+
 #define SYNC_MAX_RETRY_BACKOFF         5
 #define SYNC_LOG_REPL_RETRY_WAIT_MS    100
 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000
@@ -45,8 +46,8 @@ extern "C" {
 
 #define SYNC_HEARTBEAT_SLOW_MS       1500
 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500
-#define SYNC_SNAP_RESEND_MS          1000 * 60
-#define SYNC_SNAP_TIMEOUT_MS         1000 * 600
+#define SYNC_SNAP_RESEND_MS          1000 * 300
+#define SYNC_SNAP_TIMEOUT_MS         1000 * 1800
 
 #define SYNC_VND_COMMIT_MIN_MS 3000
 
@@ -219,6 +220,7 @@ typedef struct SSyncLogStore {
 
   SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore);
   SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore);
+  SyncIndex (*syncLogIndexRetention)(struct SSyncLogStore* pLogStore, int64_t bytes);
   SyncTerm (*syncLogLastTerm)(struct SSyncLogStore* pLogStore);
 
   int32_t (*syncLogAppendEntry)(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, bool forcSync);
@@ -289,6 +291,9 @@ const char* syncStr(ESyncState state);
 
 int32_t    syncNodeGetConfig(int64_t rid, SSyncCfg *cfg);
 
+// util
+int32_t syncSnapInfoDataRealloc(SSnapshot* pSnap, int32_t size);
+
 #ifdef __cplusplus
 }
 #endif

include/libs/wal/wal.h

@@ -225,6 +225,7 @@ bool walIsEmpty(SWal *);
 int64_t walGetFirstVer(SWal *);
 int64_t walGetSnapshotVer(SWal *);
 int64_t walGetLastVer(SWal *);
+int64_t walGetVerRetention(SWal *pWal, int64_t bytes);
 int64_t walGetCommittedVer(SWal *);
 int64_t walGetAppliedVer(SWal *);
 

include/os/osFile.h

@@ -63,6 +63,9 @@ typedef struct TdFile *TdFilePtr;
 #define TD_FILE_AUTO_DEL      0x0040
 #define TD_FILE_EXCL          0x0080
 #define TD_FILE_STREAM        0x0100  // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
+#define TD_FILE_WRITE_THROUGH 0x0200
+#define TD_FILE_CLOEXEC       0x0400
+
 TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions);
 TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions);
 

include/os/osString.h

@@ -47,18 +47,13 @@ typedef int32_t TdUcs4;
 #define strtof      STR_TO_F_FUNC_TAOS_FORBID
 #endif
 
-#ifdef WINDOWS
-#define tstrdup(str) _strdup(str)
-#else
-#define tstrdup(str) strdup(str)
-#endif
-
 #define tstrncpy(dst, src, size)   \
   do {                             \
     strncpy((dst), (src), (size)); \
     (dst)[(size)-1] = 0;           \
   } while (0)
 
+char   *tstrdup(const char *src);
 int32_t taosUcs4len(TdUcs4 *ucs4);
 int64_t taosStr2int64(const char *str);
 

include/util/taoserror.h

@@ -124,6 +124,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_INVALID_CFG_VALUE             TAOS_DEF_ERROR_CODE(0, 0x0133)
 
 #define TSDB_CODE_IP_NOT_IN_WHITE_LIST          TAOS_DEF_ERROR_CODE(0, 0x0134)
+#define TSDB_CODE_FAILED_TO_CONNECT_S3          TAOS_DEF_ERROR_CODE(0, 0x0135)
 
 //client
 #define TSDB_CODE_TSC_INVALID_OPERATION         TAOS_DEF_ERROR_CODE(0, 0x0200)
@@ -426,6 +427,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_VIEW_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x04A0)
 #define TSDB_CODE_MND_VIEW_NOT_EXIST            TAOS_DEF_ERROR_CODE(0, 0x04A1)
 
+//mnode-compact
+#define TSDB_CODE_MND_INVALID_COMPACT_ID        TAOS_DEF_ERROR_CODE(0, 0x04B1)
+
 
 // vnode
 // #define TSDB_CODE_VND_ACTION_IN_PROGRESS     TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

include/util/tdef.h

@@ -288,6 +288,7 @@ typedef enum ELogicConditionType {
 #define TSDB_CONN_ACTIVE_KEY_LEN 255
 
 #define TSDB_DEFAULT_PKT_SIZE 65480  // same as RPC_MAX_UDP_SIZE
+#define TSDB_SNAP_DATA_PAYLOAD_SIZE (1 * 1024 * 1024)
 
 #define TSDB_PAYLOAD_SIZE         TSDB_DEFAULT_PKT_SIZE
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120  // default payload size, greater than PATH_MAX value
@@ -305,7 +306,7 @@ typedef enum ELogicConditionType {
 #define TSDB_SYNC_APPLYQ_SIZE_LIMIT    512
 #define TSDB_SYNC_NEGOTIATION_WIN      512
 
-#define TSDB_SYNC_SNAP_BUFFER_SIZE 2048
+#define TSDB_SYNC_SNAP_BUFFER_SIZE 1024
 
 #define TSDB_TBNAME_COLUMN_INDEX     (-1)
 #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000  // maximum batch size allowed to load table meta

include/util/tlog.h

@@ -66,6 +66,7 @@ extern int32_t udfDebugFlag;
 extern int32_t smaDebugFlag;
 extern int32_t idxDebugFlag;
 extern int32_t tdbDebugFlag;
+extern int32_t sndDebugFlag;
 
 int32_t taosInitLog(const char *logName, int32_t maxFiles);
 void    taosCloseLog();

source/client/inc/clientInt.h

@@ -205,7 +205,7 @@ typedef struct SRequestSendRecvBody {
   __taos_async_fn_t queryFp;
   __taos_async_fn_t fetchFp;
   EQueryExecMode    execMode;
-  void*             param;
+  void*             interParam;
   SDataBuf          requestMsg;
   int64_t           queryJob;  // query job, created according to sql query DAG.
   int32_t           subplanNum;
@@ -287,6 +287,7 @@ typedef struct SRequestObj {
 typedef struct SSyncQueryParam {
   tsem_t       sem;
   SRequestObj* pRequest;
+  void*        userParam;
 } SSyncQueryParam;
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
@@ -420,6 +421,7 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
 int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
 void    returnToUser(SRequestObj* pRequest);
 void    stopAllQueries(SRequestObj *pRequest);
+void    doRequestCallback(SRequestObj* pRequest, int32_t code);
 void    freeQueryParam(SSyncQueryParam* param);
 
 #ifdef TD_ENTERPRISE

source/client/inc/clientSml.h

@@ -65,14 +65,32 @@ extern "C" {
 #define IS_INVALID_COL_LEN(len)   ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN)
 #define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN)
 
-//#define TS        "_ts"
-//#define TS_LEN    3
 #define VALUE     "_value"
-#define VALUE_LEN 6
+#define VALUE_LEN (sizeof(VALUE)-1)
 
 #define OTD_JSON_FIELDS_NUM     4
 #define MAX_RETRY_TIMES 10
-typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType;
+
+#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
+
+#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
+&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
+
+#define IS_SAME_KEY (maxKV->type == kv->type && maxKV->keyLen == kv->keyLen && memcmp(maxKV->key, kv->key, kv->keyLen) == 0)
+
+#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
+  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
+
+#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
+
+#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
+  for (int i = 1; i < keyLen; ++i) {         \
+    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
+      MOVE_FORWARD_ONE(key + i, keyLen - i); \
+      keyLen--;                              \
+    }                                        \
+  }
 
 typedef enum {
   SCHEMA_ACTION_NULL,
@@ -83,18 +101,6 @@ typedef enum {
   SCHEMA_ACTION_CHANGE_TAG_SIZE,
 } ESchemaAction;
 
-typedef struct {
-  const void  *key;
-  int32_t      keyLen;
-  void        *value;
-  bool         used;
-}Node;
-
-typedef struct NodeList{
-  Node             data;
-  struct NodeList* next;
-}NodeList;
-
 typedef struct {
   char *measure;
   char *tags;
@@ -117,7 +123,6 @@ typedef struct {
   int32_t     sTableNameLen;
   char        childTableName[TSDB_TABLE_NAME_LEN];
   uint64_t    uid;
-//  void       *key;        // for openTsdb
 
   SArray *tags;
 
@@ -161,7 +166,8 @@ typedef struct {
 typedef struct {
   int64_t id;
 
-  SMLProtocolType protocol;
+  TSDB_SML_PROTOCOL_TYPE protocol;
+
   int8_t          precision;
   bool            reRun;
   bool            dataFormat;  // true means that the name and order of keys in each line are the same(only for influx protocol)
@@ -201,29 +207,8 @@ typedef struct {
   bool         needModifySchema;
 } SSmlHandle;
 
-#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0)
-
-#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \
-&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0)
-
-#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0)
-
-#define IS_SLASH_LETTER_IN_MEASUREMENT(sql)                                                           \
-  (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE))
-
-#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len))
-
-#define PROCESS_SLASH_IN_MEASUREMENT(key, keyLen)           \
-  for (int i = 1; i < keyLen; ++i) {         \
-    if (IS_SLASH_LETTER_IN_MEASUREMENT(key + i)) {          \
-      MOVE_FORWARD_ONE(key + i, keyLen - i); \
-      keyLen--;                              \
-    }                                        \
-  }
-
-extern int64_t smlFactorNS[3];
-extern int64_t smlFactorS[3];
+extern int64_t smlFactorNS[];
+extern int64_t smlFactorS[];
 
 typedef int32_t (*_equal_fn_sml)(const void *, const void *);
 
@@ -231,16 +216,10 @@ SSmlHandle   *smlBuildSmlInfo(TAOS *taos);
 void          smlDestroyInfo(SSmlHandle *info);
 int           smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset);
 int           smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset);
-//SArray       *smlJsonParseTags(char *start, char *end);
 bool          smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg);
-//void*         nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn);
-//int           nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn);
-//int           nodeListSize(NodeList* list);
-bool          smlDoubleToInt64OverFlow(double num);
 int32_t       smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2);
 bool          smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg);
 int64_t       smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision);
-int8_t        smlGetTsTypeByLen(int32_t len);
 SSmlTableInfo*    smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen);
 SSmlSTableMeta*   smlBuildSTableMeta(bool isDataFormat);
 int32_t           smlSetCTableName(SSmlTableInfo *oneTable);
@@ -258,7 +237,40 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements);
 int32_t smlParseJSON(SSmlHandle *info, char *payload);
 
-void    smlStrReplace(char* src, int32_t len);
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement);
+bool            isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+bool            isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv);
+int32_t         smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements);
+int32_t         smlJoinMeasureTag(SSmlLineInfo *elements);
+void            smlBuildTsKv(SSmlKv *kv, int64_t ts);
+int32_t         smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv);
+int32_t         smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs);
+
+static inline bool smlDoubleToInt64OverFlow(double num) {
+  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
+  return false;
+}
+
+static inline void smlStrReplace(char* src, int32_t len){
+  if (!tsSmlDot2Underline) return;
+  for(int i = 0; i < len; i++){
+    if(src[i] == '.'){
+      src[i] = '_';
+    }
+  }
+}
+
+static inline int8_t smlGetTsTypeByLen(int32_t len) {
+  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
+    return TSDB_TIME_PRECISION_SECONDS;
+  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
+    return TSDB_TIME_PRECISION_MILLI;
+  } else {
+    return -1;
+  }
+}
+
 #ifdef __cplusplus
 }
 #endif

source/client/src/clientEnv.c

@@ -316,6 +316,15 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return NULL;
   }
+  SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (interParam == NULL) {
+    doDestroyRequest(pRequest);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
+  tsem_init(&interParam->sem, 0, 0);
+  interParam->pRequest = pRequest;
+  pRequest->body.interParam = interParam;
 
   pRequest->resType = RES_TYPE__QUERY;
   pRequest->requestId = reqid == 0 ? generateRequestId() : reqid;
@@ -437,12 +446,10 @@ void doDestroyRequest(void *p) {
     deregisterRequest(pRequest);
   }
 
-  if (pRequest->syncQuery) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
+  if (pRequest->body.interParam) {
+    tsem_destroy(&((SSyncQueryParam *)pRequest->body.interParam)->sem);
   }
+  taosMemoryFree(pRequest->body.interParam);
 
   qDestroyQuery(pRequest->pQuery);
   nodesDestroyAllocator(pRequest->allocatorRefId);
@@ -555,6 +562,9 @@ static void *tscCrashReportThreadFp(void *param) {
         if (pFile) {
           taosReleaseCrashLogFile(pFile, false);
           pFile = NULL;
+
+          taosMsleep(sleepTime);
+          loopTimes = 0;
           continue;
         }
       } else {
@@ -729,7 +739,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
         return -1;
       }
       newstr[0] = '"';
-      strncpy(newstr+1, str, len);
+      memcpy(newstr+1, str, len);
       newstr[len + 1] = '"';
       newstr[len + 2] = '\0';
       str = newstr;

source/client/src/clientImpl.c

@@ -196,21 +196,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   (*pRequest)->sqlLen = sqlLen;
   (*pRequest)->validateOnly = validateSql;
 
-  SSyncQueryParam* newpParam = NULL;
-  if (param == NULL) {
-    newpParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
-    if (newpParam == NULL) {
-      destroyRequest(*pRequest);
-      *pRequest = NULL;
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-
-    tsem_init(&newpParam->sem, 0, 0);
-    newpParam->pRequest = (*pRequest);
-    param = newpParam;
-  }
-
-  (*pRequest)->body.param = param;
+  ((SSyncQueryParam*)(*pRequest)->body.interParam)->userParam = param;
 
   STscObj* pTscObj = (*pRequest)->pTscObj;
   int32_t  err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self,
@@ -218,7 +204,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
   if (err) {
     tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
              (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-    freeQueryParam(newpParam);
     destroyRequest(*pRequest);
     *pRequest = NULL;
     return TSDB_CODE_OUT_OF_MEMORY;
@@ -230,7 +215,6 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param,
         nodesCreateAllocator((*pRequest)->requestId, tsQueryNodeChunkSize, &((*pRequest)->allocatorRefId))) {
       tscError("%" PRId64 " failed to create node allocator, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s",
                (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql);
-      freeQueryParam(newpParam);
       destroyRequest(*pRequest);
       *pRequest = NULL;
       return TSDB_CODE_OUT_OF_MEMORY;
@@ -336,7 +320,7 @@ static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscOb
 void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
   SRetrieveTableRsp* pRsp = NULL;
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -358,18 +342,18 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) {
              pRequest->requestId);
   }
 
-  pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+  doRequestCallback(pRequest, code);
 }
 
 int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   if (pRequest->validateOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
   // drop table if exists not_exists_table
   if (NULL == pQuery->pCmdMsg) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -384,7 +368,7 @@ int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
   int64_t transporterId = 0;
   int32_t code = asyncSendMsgToServer(pAppInfo->pTransporter, &pMsgInfo->epSet, &transporterId, pSendMsg);
   if (code) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
   return code;
 }
@@ -913,7 +897,7 @@ void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
 void returnToUser(SRequestObj* pRequest) {
   if (pRequest->relation.userRefId == pRequest->self || 0 == pRequest->relation.userRefId) {
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
     return;
   }
 
@@ -921,7 +905,7 @@ void returnToUser(SRequestObj* pRequest) {
   if (pUserReq) {
     pUserReq->code = pRequest->code;
     // return to client
-    pUserReq->body.queryFp(pUserReq->body.param, pUserReq, pUserReq->code);
+    doRequestCallback(pUserReq, pUserReq->code);
     releaseRequest(pRequest->relation.userRefId);
     return;
   } else {
@@ -1031,7 +1015,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
     pRequest->pWrapper = NULL;
 
     // return to client
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1186,7 +1170,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
       pRequest->code = terrno;
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 
   // todo not to be released here
@@ -1199,7 +1183,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
   int32_t code = 0;
 
   if (pRequest->parseOnly) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+    doRequestCallback(pRequest, 0);
     return;
   }
 
@@ -1233,11 +1217,11 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
     }
     case QUERY_EXEC_MODE_EMPTY_RESULT:
       pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
-      pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
+      doRequestCallback(pRequest, 0);
       break;
     default:
       tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
-      pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
+      doRequestCallback(pRequest, -1);
       break;
   }
 }
@@ -1703,8 +1687,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
 }
 
 static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) {
-  SSyncQueryParam* pParam = param;
-  tsem_post(&pParam->sem);
+  tsem_t* sem = param;
+  tsem_post(sem);
 }
 
 void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) {
@@ -1722,10 +1706,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU
 
     // convert ucs4 to native multi-bytes string
     pResultInfo->convertUcs4 = convertUcs4;
-
-    SSyncQueryParam* pParam = pRequest->body.param;
-    taos_fetch_rows_a(pRequest, syncFetchFn, pParam);
-    tsem_wait(&pParam->sem);
+    tsem_t sem;
+    tsem_init(&sem, 0, 0);
+    taos_fetch_rows_a(pRequest, syncFetchFn, &sem);
+    tsem_wait(&sem);
+    tsem_destroy(&sem);
   }
 
   if (pResultInfo->numOfRows == 0 || pRequest->code != TSDB_CODE_SUCCESS) {
@@ -2492,6 +2477,10 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   tscDebug("taos_query start with sql:%s", sql);
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (NULL == param) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly);
@@ -2501,9 +2490,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
+  taosMemoryFree(param);
 
   tscDebug("taos_query end with sql:%s", sql);
 
@@ -2517,19 +2505,20 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly,
   }
 
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
+  if (param == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return NULL;
+  }
   tsem_init(&param->sem, 0, 0);
 
   taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid);
   tsem_wait(&param->sem);
-
   SRequestObj* pRequest = NULL;
   if (param->pRequest != NULL) {
     param->pRequest->syncQuery = true;
     pRequest = param->pRequest;
-  } else {
-    taosMemoryFree(param);
   }
-
+  taosMemoryFree(param);
   return pRequest;
 }
 
@@ -2547,13 +2536,13 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
   if (code != TSDB_CODE_SUCCESS) {
     pRequest->code = code;
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
   if (pRequest->code != TSDB_CODE_SUCCESS) {
     taosMemoryFreeClear(pResultInfo->pData);
-    pRequest->body.fetchFp(pRequest->body.param, pRequest, 0);
+    pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, 0);
     return;
   }
 
@@ -2574,20 +2563,12 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
     atomic_add_fetch_64((int64_t*)&pActivity->fetchBytes, pRequest->body.resInfo.payloadLen);
   }
 
-  pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows);
+  pRequest->body.fetchFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, pResultInfo->numOfRows);
 }
 
 void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param) {
-  if (pRequest->syncQuery && pRequest->body.param != param) {
-    if (pRequest->body.param) {
-      tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem);
-    }
-    taosMemoryFree(pRequest->body.param);
-    pRequest->syncQuery = false;
-  }
-  
   pRequest->body.fetchFp = fp;
-  pRequest->body.param = param;
+  ((SSyncQueryParam *)pRequest->body.interParam)->userParam = param;
 
   SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
 
@@ -2625,6 +2606,10 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
   schedulerFetchRows(pRequest->body.queryJob, &req);
 }
 
+void doRequestCallback(SRequestObj* pRequest, int32_t code) {
+  pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
+}
+
 int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
 #ifndef TD_ENTERPRISE
   return TSDB_CODE_SUCCESS;
@@ -2633,4 +2618,3 @@ int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool pa
 #endif
 }
 
-

source/client/src/clientMain.c

@@ -876,6 +876,7 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
     code = 0;
   }
   taosThreadMutexUnlock(&pTscObj->mutex);
+  releaseTscObj(*(int64_t *)taos);
   return code;
 }
 
@@ -1094,7 +1095,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1111,7 +1112,7 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest)
     pRequest->pWrapper = NULL;
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1209,7 +1210,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
     terrno = code;
     pRequest->code = code;
     tscDebug("call sync query cb with code: %s", tstrerror(code));
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
     return;
   }
 
@@ -1241,7 +1242,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) {
 
     terrno = code;
     pRequest->code = code;
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   }
 }
 
@@ -1548,12 +1549,12 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
 
   conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
 
-  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.param, NULL);
+  code = catalogAsyncGetAllMeta(pCtg, &conn, &catalogReq, syncCatalogFn, pRequest->body.interParam, NULL);
   if (code) {
     goto _return;
   }
 
-  SSyncQueryParam *pParam = pRequest->body.param;
+  SSyncQueryParam *pParam = pRequest->body.interParam;
   tsem_wait(&pParam->sem);
 
 _return:

source/client/src/clientMsgHandler.c

@@ -16,6 +16,7 @@
 #include "catalog.h"
 #include "clientInt.h"
 #include "clientLog.h"
+#include "cmdnodes.h"
 #include "os.h"
 #include "query.h"
 #include "systable.h"
@@ -41,7 +42,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
   taosMemoryFree(pMsg->pData);
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -199,7 +200,7 @@ int32_t processCreateDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   }
 
   if (pRequest->body.queryFp) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -235,7 +236,8 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
     setErrno(pRequest, code);
 
     if (pRequest->body.queryFp != NULL) {
-      pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+      doRequestCallback(pRequest, pRequest->code);
+
     } else {
       tsem_post(&pRequest->body.rspSem);
     }
@@ -299,7 +301,7 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, pRequest->code);
+    doRequestCallback(pRequest, pRequest->code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -343,7 +345,7 @@ int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -380,7 +382,7 @@ int32_t processDropDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -420,7 +422,7 @@ int32_t processAlterStbRsp(void* param, SDataBuf* pMsg, int32_t code) {
       }
     }
 
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -534,7 +536,119 @@ int32_t processShowVariablesRsp(void* param, SDataBuf* pMsg, int32_t code) {
   taosMemoryFree(pMsg->pEpSet);
 
   if (pRequest->body.queryFp != NULL) {
-    pRequest->body.queryFp(pRequest->body.param, pRequest, code);
+    doRequestCallback(pRequest, code);
+  } else {
+    tsem_post(&pRequest->body.rspSem);
+  }
+  return code;
+}
+
+static int32_t buildCompactDbBlock(SCompactDbRsp* pRsp, SSDataBlock** block) {
+  SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
+  pBlock->info.hasVarCol = true;
+
+  pBlock->pDataBlock = taosArrayInit(COMPACT_DB_RESULT_COLS, sizeof(SColumnInfoData));
+
+  SColumnInfoData infoData = {0};
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = COMPACT_DB_RESULT_FIELD1_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_INT;
+  infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
+  infoData.info.bytes = COMPACT_DB_RESULT_FIELD3_LEN;
+  taosArrayPush(pBlock->pDataBlock, &infoData);
+
+  blockDataEnsureCapacity(pBlock, 1);
+
+  SColumnInfoData* pResultCol = taosArrayGet(pBlock->pDataBlock, 0);
+  SColumnInfoData* pIdCol = taosArrayGet(pBlock->pDataBlock, 1);
+  SColumnInfoData* pReasonCol = taosArrayGet(pBlock->pDataBlock, 2);
+  char result[COMPACT_DB_RESULT_FIELD1_LEN] = {0};
+  char reason[COMPACT_DB_RESULT_FIELD3_LEN] = {0};
+  if (pRsp->bAccepted) {
+    STR_TO_VARSTR(result, "accepted");
+    colDataSetVal(pResultCol, 0, result, false);
+    colDataSetVal(pIdCol, 0, (void*)&pRsp->compactId, false);
+    STR_TO_VARSTR(reason, "success");
+    colDataSetVal(pReasonCol, 0, reason, false);
+  } else {
+    STR_TO_VARSTR(result, "rejected");
+    colDataSetVal(pResultCol, 0, result, false);
+    colDataSetNULL(pIdCol, 0);
+    STR_TO_VARSTR(reason, "compaction is ongoing");
+    colDataSetVal(pReasonCol, 0, reason, false);
+  }
+  pBlock->info.rows = 1;
+
+  *block = pBlock;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetrieveTableRsp** pRsp) {
+  SSDataBlock* pBlock = NULL;
+  int32_t      code = buildCompactDbBlock(pCompactDb, &pBlock);
+  if (code) {
+    return code;
+  }
+
+  size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock);
+  *pRsp = taosMemoryCalloc(1, rspSize);
+  if (NULL == *pRsp) {
+    blockDataDestroy(pBlock);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  (*pRsp)->useconds = 0;
+  (*pRsp)->completed = 1;
+  (*pRsp)->precision = 0;
+  (*pRsp)->compressed = 0;
+  (*pRsp)->compLen = 0;
+  (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
+  (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
+
+  int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS);
+  blockDataDestroy(pBlock);
+
+  if (len != rspSize - sizeof(SRetrieveTableRsp)) {
+    uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
+           (uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
+    return TSDB_CODE_TSC_INVALID_INPUT;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t processCompactDbRsp(void* param, SDataBuf* pMsg, int32_t code) {
+  SRequestObj* pRequest = param;
+  if (code != TSDB_CODE_SUCCESS) {
+    setErrno(pRequest, code);
+  } else {
+    SCompactDbRsp  rsp = {0};
+    SRetrieveTableRsp* pRes = NULL;
+    code = tDeserializeSCompactDbRsp(pMsg->pData, pMsg->len, &rsp);
+    if (TSDB_CODE_SUCCESS == code) {
+      code = buildRetriveTableRspForCompactDb(&rsp, &pRes);
+    }
+    if (TSDB_CODE_SUCCESS == code) {
+      code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, true);
+    }
+
+    if (code != 0) {
+      taosMemoryFree(pRes);
+    }
+  }
+
+  taosMemoryFree(pMsg->pData);
+  taosMemoryFree(pMsg->pEpSet);
+
+  if (pRequest->body.queryFp != NULL) {
+    pRequest->body.queryFp(((SSyncQueryParam *)pRequest->body.interParam)->userParam, pRequest, code);
   } else {
     tsem_post(&pRequest->body.rspSem);
   }
@@ -557,6 +671,8 @@ __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) {
       return processAlterStbRsp;
     case TDMT_MND_SHOW_VARIABLES:
       return processShowVariablesRsp;
+    case TDMT_MND_COMPACT_DB:
+      return processCompactDbRsp;  
     default:
       return genericRspCallback;
   }

source/client/src/clientSml.c

@@ -20,75 +20,93 @@
 
 #include "clientSml.h"
 
-int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL};
-int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
-int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL};
+#define RETURN_FALSE                                 \
+  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
+  return false;
 
-//void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) {
-//  NodeList *tmp = list;
-//  while (tmp) {
-//    if (fn == NULL) {
-//      if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return tmp->data.value;
-//      }
-//    } else {
-//      if (tmp->data.used && fn(tmp->data.key, key) == 0) {
-//        return tmp->data.value;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  return NULL;
-//}
-//
-//int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) {
-//  NodeList *tmp = *list;
-//  while (tmp) {
-//    if (!tmp->data.used) break;
-//    if (fn == NULL) {
-//      if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) {
-//        return -1;
-//      }
-//    } else {
-//      if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) {
-//        return -1;
-//      }
-//    }
-//
-//    tmp = tmp->next;
-//  }
-//  if (tmp) {
-//    tmp->data.key = key;
-//    tmp->data.keyLen = len;
-//    tmp->data.value = value;
-//    tmp->data.used = true;
-//  } else {
-//    NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList));
-//    if (newNode == NULL) {
-//      return -1;
-//    }
-//    newNode->data.key = key;
-//    newNode->data.keyLen = len;
-//    newNode->data.value = value;
-//    newNode->data.used = true;
-//    newNode->next = *list;
-//    *list = newNode;
-//  }
-//  return 0;
-//}
-//
-//int nodeListSize(NodeList *list) {
-//  int cnt = 0;
-//  while (list) {
-//    if (list->data.used)
-//      cnt++;
-//    else
-//      break;
-//    list = list->next;
-//  }
-//  return cnt;
-//}
+#define SET_DOUBLE                     \
+  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
+  kvVal->d = result;
+
+#define SET_FLOAT                                                                              \
+  if (!IS_VALID_FLOAT(result)) {                                                               \
+    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
+    return false;                                                                              \
+  }                                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
+  kvVal->f = (float)result;
+
+#define SET_BIGINT                                                                                       \
+  errno = 0;                                                                                             \
+  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
+  if (errno == ERANGE) {                                                                                 \
+    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
+    return false;                                                                                        \
+  }                                                                                                      \
+  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
+  kvVal->i = tmp;
+
+#define SET_INT                                                                    \
+  if (!IS_VALID_INT(result)) {                                                     \
+    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
+    return false;                                                                  \
+  }                                                                                \
+  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
+  kvVal->i = result;
+
+#define SET_SMALL_INT                                                          \
+  if (!IS_VALID_SMALLINT(result)) {                                            \
+    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
+    return false;                                                              \
+  }                                                                            \
+  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
+  kvVal->i = result;
+
+#define SET_UBIGINT                                                                             \
+  errno = 0;                                                                                    \
+  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
+  if (errno == ERANGE || result < 0) {                                                          \
+    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
+    return false;                                                                               \
+  }                                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
+  kvVal->u = tmp;
+
+#define SET_UINT                                                                  \
+  if (!IS_VALID_UINT(result)) {                                                   \
+    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
+  kvVal->u = result;
+
+#define SET_USMALL_INT                                                            \
+  if (!IS_VALID_USMALLINT(result)) {                                              \
+    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
+    return false;                                                                 \
+  }                                                                               \
+  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
+  kvVal->u = result;
+
+#define SET_TINYINT                                                       \
+  if (!IS_VALID_TINYINT(result)) {                                        \
+    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
+    return false;                                                         \
+  }                                                                       \
+  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
+  kvVal->i = result;
+
+#define SET_UTINYINT                                                            \
+  if (!IS_VALID_UTINYINT(result)) {                                             \
+    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
+    return false;                                                               \
+  }                                                                             \
+  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
+  kvVal->u = result;
+
+int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
+int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
+int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
 
 static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){
   SUserAuthInfo pAuth = {0};
@@ -109,19 +127,6 @@ static int32_t smlCheckAuth(SSmlHandle *info,  SRequestConnInfo* conn, const cha
   return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
 
 }
-inline bool smlDoubleToInt64OverFlow(double num) {
-  if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true;
-  return false;
-}
-
-void smlStrReplace(char* src, int32_t len){
-  if (!tsSmlDot2Underline) return;
-  for(int i = 0; i < len; i++){
-    if(src[i] == '.'){
-      src[i] = '_';
-    }
-  }
-}
 
 int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
   if (pBuf->buf) {
@@ -155,16 +160,6 @@ int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, u
   return convertTimePrecision(tsInt64, fromPrecision, toPrecision);
 }
 
-int8_t smlGetTsTypeByLen(int32_t len) {
-  if (len == TSDB_TIME_PRECISION_SEC_DIGITS) {
-    return TSDB_TIME_PRECISION_SECONDS;
-  } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) {
-    return TSDB_TIME_PRECISION_MILLI;
-  } else {
-    return -1;
-  }
-}
-
 SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) {
   SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1);
   if (!tag) {
@@ -180,11 +175,6 @@ SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measu
     goto cleanup;
   }
 
-  //  tag->tags = taosArrayInit(16, sizeof(SSmlKv));
-  //  if (tag->tags == NULL) {
-  //    uError("SML:smlBuildTableInfo failed to allocate memory");
-  //    goto cleanup;
-  //  }
   return tag;
 
 cleanup:
@@ -192,6 +182,242 @@ cleanup:
   return NULL;
 }
 
+void smlBuildTsKv(SSmlKv *kv, int64_t ts){
+  kv->key = tsSmlTsDefaultName;
+  kv->keyLen = strlen(tsSmlTsDefaultName);
+  kv->type = TSDB_DATA_TYPE_TIMESTAMP;
+  kv->i = ts;
+  kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
+}
+
+SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){
+  SSmlSTableMeta* sMeta = NULL;
+  char *measure = currElement->measure;
+  int   measureLen = currElement->measureLen;
+  if (currElement->measureEscaped) {
+    measure = (char *)taosMemoryMalloc(measureLen);
+    memcpy(measure, currElement->measure, measureLen);
+    PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
+    smlStrReplace(measure, measureLen);
+  }
+  STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
+  if (currElement->measureEscaped) {
+    taosMemoryFree(measure);
+  }
+  if (pTableMeta == NULL) {
+    info->dataFormat = false;
+    info->reRun = true;
+    terrno = TSDB_CODE_SUCCESS;
+    return sMeta;
+  }
+  sMeta = smlBuildSTableMeta(info->dataFormat);
+  if(sMeta == NULL){
+    taosMemoryFreeClear(pTableMeta);
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return sMeta;
+  }
+  sMeta->tableMeta = pTableMeta;
+  taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
+  for (int i = 1; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
+    SSchema *col = pTableMeta->schema + i;
+    SSmlKv   kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
+    if (col->type == TSDB_DATA_TYPE_NCHAR) {
+      kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
+    } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) {
+      kv.length = col->bytes - VARSTR_HEADER_SIZE;
+    } else{
+      kv.length = col->bytes;
+    }
+
+    if(i < pTableMeta->tableInfo.numOfColumns){
+      taosArrayPush(sMeta->cols, &kv);
+    }else{
+      taosArrayPush(sMeta->tags, &kv);
+    }
+  }
+  return sMeta;
+}
+
+bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  // cnt begin 0, add ts so + 2
+  if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  // bind data
+  int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, cnt + 1);
+  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+    uDebug("smlBuildCol error, retry");
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  if (cnt >= taosArrayGetSize(info->maxColKVs)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    info->dataFormat = false;
+    info->reRun = true;
+    return false;
+  }
+
+  if (unlikely(IS_VAR_DATA_TYPE(kv->type) && kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+}
+
+bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
+  if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
+    goto END;
+  }
+
+  if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
+    goto END;
+  }
+  SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
+
+  if (unlikely(!IS_SAME_KEY)) {
+    goto END;
+  }
+
+  if (unlikely(kv->length > maxKV->length)) {
+    maxKV->length = kv->length;
+    info->needModifySchema = true;
+  }
+  return true;
+
+END:
+  info->dataFormat = false;
+  info->reRun = true;
+  return false;
+}
+
+int32_t smlJoinMeasureTag(SSmlLineInfo *elements){
+  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
+  if(elements->measureTag == NULL){
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+  memcpy(elements->measureTag, elements->measure, elements->measureLen);
+  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
+  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
+  bool isSameMeasure = IS_SAME_SUPER_TABLE;
+  if(isSameMeasure) {
+    return 0;
+  }
+  SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
+
+  SSmlSTableMeta *sMeta = NULL;
+  if (unlikely(tmp == NULL)) {
+    sMeta = smlBuildSuperTableInfo(info, elements);
+    if(sMeta == NULL) return -1;
+  }else{
+    sMeta = *tmp;
+  }
+  ASSERT(sMeta != NULL);
+  info->currSTableMeta = sMeta->tableMeta;
+  info->maxTagKVs = sMeta->tags;
+  info->maxColKVs = sMeta->cols;
+  return 0;
+}
+
+int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
+  SSmlTableInfo **oneTable =
+      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
+  SSmlTableInfo *tinfo = NULL;
+  if (unlikely(oneTable == NULL)) {
+    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
+    if (unlikely(!tinfo)) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    taosHashPut(info->childTables, elements->measureTag, elements->measureTagsLen, &tinfo, POINTER_BYTES);
+
+    tinfo->tags = taosArrayDup(info->preLineTagKV, NULL);
+    for (size_t i = 0; i < taosArrayGetSize(info->preLineTagKV); i++) {
+      SSmlKv *kv = (SSmlKv *)taosArrayGet(info->preLineTagKV, i);
+      if (kv->keyEscaped) kv->key = NULL;
+      if (kv->valueEscaped) kv->value = NULL;
+    }
+
+    smlSetCTableName(tinfo);
+    getTableUid(info, elements, tinfo);
+    if (info->dataFormat) {
+      info->currSTableMeta->uid = tinfo->uid;
+      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
+      if (tinfo->tableDataCtx == NULL) {
+        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
+        smlDestroyTableInfo(&tinfo);
+        return TSDB_CODE_SML_INVALID_DATA;
+      }
+    }
+  }else{
+    tinfo = *oneTable;
+  }
+  ASSERT(tinfo != NULL);
+  if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kv, 1);
+    }
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format false, ts:%" PRId64, info->id, kvTs->i);
+    if (elements->colArray == NULL) {
+      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
+    }
+    taosArrayPush(elements->colArray, kvTs);
+    taosArrayPush(elements->colArray, kv);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
+  if (info->dataFormat) {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
+    int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
+    if (ret == TSDB_CODE_SUCCESS) {
+      ret = smlBuildRow(info->currTableDataCtx);
+    }
+
+    clearColValArraySml(info->currTableDataCtx->pValues);
+    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
+      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
+      return ret;
+    }
+  } else {
+    uDebug("SML:0x%" PRIx64 " smlParseEndLine format false, ts:%" PRId64, info->id, kvTs->i);
+    taosArraySet(elements->colArray, 0, kvTs);
+  }
+  info->preLine = *elements;
+
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t smlParseTableName(SArray *tags, char *childTableName) {
   bool autoChildName = false;
   size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
@@ -328,98 +554,6 @@ cleanup:
   return NULL;
 }
 
-// uint16_t smlCalTypeSum(char* endptr, int32_t left){
-//   uint16_t sum = 0;
-//   for(int i = 0; i < left; i++){
-//     sum += endptr[i];
-//   }
-//   return sum;
-// }
-
-#define RETURN_FALSE                                 \
-  smlBuildInvalidDataMsg(msg, "invalid data", pVal); \
-  return false;
-
-#define SET_DOUBLE                     \
-  kvVal->type = TSDB_DATA_TYPE_DOUBLE; \
-  kvVal->d = result;
-
-#define SET_FLOAT                                                                              \
-  if (!IS_VALID_FLOAT(result)) {                                                               \
-    smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \
-    return false;                                                                              \
-  }                                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_FLOAT;                                                          \
-  kvVal->f = (float)result;
-
-#define SET_BIGINT                                                                                       \
-  errno = 0;                                                                                             \
-  int64_t tmp = taosStr2Int64(pVal, &endptr, 10);                                                        \
-  if (errno == ERANGE) {                                                                                 \
-    smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \
-    return false;                                                                                        \
-  }                                                                                                      \
-  kvVal->type = TSDB_DATA_TYPE_BIGINT;                                                                   \
-  kvVal->i = tmp;
-
-#define SET_INT                                                                    \
-  if (!IS_VALID_INT(result)) {                                                     \
-    smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \
-    return false;                                                                  \
-  }                                                                                \
-  kvVal->type = TSDB_DATA_TYPE_INT;                                                \
-  kvVal->i = result;
-
-#define SET_SMALL_INT                                                          \
-  if (!IS_VALID_SMALLINT(result)) {                                            \
-    smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \
-    return false;                                                              \
-  }                                                                            \
-  kvVal->type = TSDB_DATA_TYPE_SMALLINT;                                       \
-  kvVal->i = result;
-
-#define SET_UBIGINT                                                                             \
-  errno = 0;                                                                                    \
-  uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);                                             \
-  if (errno == ERANGE || result < 0) {                                                          \
-    smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \
-    return false;                                                                               \
-  }                                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UBIGINT;                                                         \
-  kvVal->u = tmp;
-
-#define SET_UINT                                                                  \
-  if (!IS_VALID_UINT(result)) {                                                   \
-    smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_UINT;                                              \
-  kvVal->u = result;
-
-#define SET_USMALL_INT                                                            \
-  if (!IS_VALID_USMALLINT(result)) {                                              \
-    smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \
-    return false;                                                                 \
-  }                                                                               \
-  kvVal->type = TSDB_DATA_TYPE_USMALLINT;                                         \
-  kvVal->u = result;
-
-#define SET_TINYINT                                                       \
-  if (!IS_VALID_TINYINT(result)) {                                        \
-    smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \
-    return false;                                                         \
-  }                                                                       \
-  kvVal->type = TSDB_DATA_TYPE_TINYINT;                                   \
-  kvVal->i = result;
-
-#define SET_UTINYINT                                                            \
-  if (!IS_VALID_UTINYINT(result)) {                                             \
-    smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \
-    return false;                                                               \
-  }                                                                             \
-  kvVal->type = TSDB_DATA_TYPE_UTINYINT;                                        \
-  kvVal->u = result;
-
 bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) {
   const char *pVal = kvVal->value;
   int32_t     len = kvVal->length;
@@ -765,8 +899,6 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
   return TSDB_CODE_SUCCESS;
 }
 
-// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData,
-//                               int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){
 static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta,
                               ESchemaAction action) {
   SRequestObj   *pRequest = NULL;
@@ -1121,35 +1253,6 @@ end:
   return code;
 }
 
-/*
-static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){
-  for(int i = 0; i < taosArrayGetSize(tags); i++) {
-    SSmlKv *tag = taosArrayGet(tags, i);
-    if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) {
-      smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key);
-      return TSDB_CODE_TSC_DUP_NAMES;
-    }
-  }
-  return TSDB_CODE_SUCCESS;
-}
-
-static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) {
-  SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
-  int ret = smlCheckDupUnit(dumplicateKey, cols, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-  ret = smlCheckDupUnit(dumplicateKey, tags, msg);
-  if(ret != TSDB_CODE_SUCCESS){
-    goto end;
-  }
-
-  end:
-  taosHashCleanup(dumplicateKey);
-  return ret;
-}
-*/
-
 static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) {
   for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) {
     SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i);
@@ -1206,11 +1309,6 @@ void smlDestroyTableInfo(void *para) {
     taosHashCleanup(kvHash);
   }
 
-  //  if (info->parseJsonByLib) {
-  //    SSmlLineInfo *key = (SSmlLineInfo *)(tag->key);
-  //    if (key != NULL) taosMemoryFree(key->tags);
-  //  }
-  //  taosMemoryFree(tag->key);
   taosArrayDestroy(tag->cols);
   taosArrayDestroyEx(tag->tags, freeSSmlKv);
   taosMemoryFree(tag);
@@ -1228,21 +1326,6 @@ void smlDestroyInfo(SSmlHandle *info) {
   if (!info) return;
   qDestroyQuery(info->pQuery);
 
-  // destroy info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // destroy info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
-
-  // destroy info->pVgHash
   taosHashCleanup(info->pVgHash);
   taosHashCleanup(info->childTables);
   taosHashCleanup(info->superTables);
@@ -1268,7 +1351,9 @@ void smlDestroyInfo(SSmlHandle *info) {
       if (info->parseJsonByLib) {
         taosMemoryFree(info->lines[i].tags);
       }
-      if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag);
+      if (info->lines[i].measureTagsLen != 0 && info->protocol != TSDB_SML_LINE_PROTOCOL) {
+        taosMemoryFree(info->lines[i].measureTag);
+      }
     }
     taosMemoryFree(info->lines);
   }
@@ -1399,11 +1484,6 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
         return ret;
       }
     } else {
-      //      ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf);
-      //      if (ret != TSDB_CODE_SUCCESS) {
-      //        uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id);
-      //        return ret;
-      //      }
       uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
              info->lineNum);
       SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
@@ -1526,30 +1606,17 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
   uDebug(
       "SML:0x%" PRIx64
       " smlInsertLines result, code:%d, msg:%s, lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \
-        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64
-      "",
+        parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64,
       info->id, info->cost.code, tstrerror(info->cost.code), info->cost.lineNum, info->cost.numOfSTables,
       info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables,
       info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
       info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
       info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
+
 }
 
 int32_t smlClearForRerun(SSmlHandle *info) {
   info->reRun = false;
-  // clear info->childTables
-//  SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL);
-//  while (oneTable) {
-//    smlDestroyTableInfo(info, *oneTable);
-//    oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable);
-//  }
-
-  // clear info->superTables
-//  SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL);
-//  while (oneSTable) {
-//    smlDestroySTableMeta(*oneSTable);
-//    oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable);
-//  }
 
   taosHashClear(info->childTables);
   taosHashClear(info->superTables);

source/client/src/clientSmlJson.c

@@ -29,199 +29,6 @@
       (start)++;                 \
   }
 
-// SArray *smlJsonParseTags(char *start, char *end){
-//   SArray *tags = taosArrayInit(4, sizeof(SSmlKv));
-//   while(start < end){
-//     SSmlKv kv = {0};
-//     kv.type = TSDB_DATA_TYPE_NCHAR;
-//     bool isInQuote = false;
-//     while(start < end){
-//       if(unlikely(!isInQuote && *start == '"')){
-//         start++;
-//         kv.key = start;
-//         isInQuote = true;
-//         continue;
-//       }
-//       if(unlikely(isInQuote && *start == '"')){
-//         kv.keyLen = start - kv.key;
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//     bool hasColon = false;
-//     while(start < end){
-//       if(unlikely(!hasColon && *start == ':')){
-//         start++;
-//         hasColon = true;
-//         continue;
-//       }
-//       if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){
-//         kv.value = start;
-//         start++;
-//         continue;
-//       }
-//
-//       if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){
-//         kv.length = start - kv.value;
-//         taosArrayPush(tags, &kv);
-//         start++;
-//         break;
-//       }
-//       start++;
-//     }
-//   }
-//   return tags;
-// }
-
-// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) {
-//   int32_t ret = TSDB_CODE_SUCCESS;
-//
-//   if(is_same_child_table_telnet(elements, &info->preLine) == 0){
-//     return TSDB_CODE_SUCCESS;
-//   }
-//
-//   bool isSameMeasure = IS_SAME_SUPER_TABLE;
-//
-//   int     cnt = 0;
-//   SArray *preLineKV = info->preLineTagKV;
-//   bool    isSuperKVInit = true;
-//   SArray *superKV = NULL;
-//   if(info->dataFormat){
-//     if(unlikely(!isSameMeasure)){
-//       SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//       elements->measureLen, NULL);
-//
-//       if(unlikely(sMeta == NULL)){
-//         sMeta = smlBuildSTableMeta(info->dataFormat);
-//         STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-//         sMeta->tableMeta = pTableMeta;
-//         if(pTableMeta == NULL){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL);
-//       }
-//       info->currSTableMeta = sMeta->tableMeta;
-//       superKV = sMeta->tags;
-//
-//       if(unlikely(taosArrayGetSize(superKV) == 0)){
-//         isSuperKVInit = false;
-//       }
-//       taosArraySetSize(preLineKV, 0);
-//     }
-//   }else{
-//     taosArraySetSize(preLineKV, 0);
-//   }
-//
-//   SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen);
-//   int32_t tagNum = taosArrayGetSize(tags);
-//   if (tagNum == 0) {
-//     uError("SML:tag is empty:%s", elements->tags)
-//     taosArrayDestroy(tags);
-//     return TSDB_CODE_SML_INVALID_DATA;
-//   }
-//   for (int32_t i = 0; i < tagNum; ++i) {
-//     SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i);
-//
-//     if(info->dataFormat){
-//       if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){
-//         info->dataFormat = false;
-//         info->reRun      = true;
-//         taosArrayDestroy(tags);
-//         return TSDB_CODE_SUCCESS;
-//       }
-//
-//       if(isSameMeasure){
-//         if(unlikely(cnt >= taosArrayGetSize(preLineKV))) {
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//         SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt);
-//         if(unlikely(kv.length > preKV->length)){
-//           preKV->length = kv.length;
-//           SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure,
-//           elements->measureLen, NULL);
-//            if(unlikely(NULL == tableMeta)){
-//              uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id);
-//              return TSDB_CODE_SML_INTERNAL_ERROR;
-//            }
-//
-//           SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt);
-//           oldKV->length = kv.length;
-//           info->needModifySchema = true;
-//         }
-//         if(unlikely(!IS_SAME_KEY)){
-//           info->dataFormat = false;
-//           info->reRun      = true;
-//           taosArrayDestroy(tags);
-//           return TSDB_CODE_SUCCESS;
-//         }
-//       }else{
-//         if(isSuperKVInit){
-//           if(unlikely(cnt >= taosArrayGetSize(superKV))) {
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//           SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt);
-//           if(unlikely(kv.length > preKV->length)) {
-//             preKV->length = kv.length;
-//           }else{
-//             kv.length = preKV->length;
-//           }
-//           info->needModifySchema = true;
-//
-//           if(unlikely(!IS_SAME_KEY)){
-//             info->dataFormat = false;
-//             info->reRun      = true;
-//             taosArrayDestroy(tags);
-//             return TSDB_CODE_SUCCESS;
-//           }
-//         }else{
-//           taosArrayPush(superKV, &kv);
-//         }
-//         taosArrayPush(preLineKV, &kv);
-//       }
-//     }else{
-//       taosArrayPush(preLineKV, &kv);
-//     }
-//     cnt++;
-//   }
-//   taosArrayDestroy(tags);
-//
-//   SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES,
-//   is_same_child_table_telnet); if (unlikely(tinfo == NULL)) {
-//     tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-//     if (unlikely(!tinfo)) {
-//       return TSDB_CODE_OUT_OF_MEMORY;
-//     }
-//     tinfo->tags = taosArrayDup(preLineKV, NULL);
-//
-//     smlSetCTableName(tinfo);
-//     if (info->dataFormat) {
-//       info->currSTableMeta->uid = tinfo->uid;
-//       tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-//       if (tinfo->tableDataCtx == NULL) {
-//         smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-//         return TSDB_CODE_SML_INVALID_DATA;
-//       }
-//     }
-//
-//     SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-//     *key = *elements;
-//     tinfo->key = key;
-//     nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet);
-//   }
-//   if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx;
-//
-//   return ret;
-// }
-
 static char *smlJsonGetObj(char *payload) {
   int  leftBracketCnt = 0;
   bool isInQuote = false;
@@ -659,12 +466,7 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
       break;
     }
     case cJSON_String: {
-      /* set default JSON type to binary/nchar according to
-       * user configured parameter tsDefaultJSONStrType
-       */
-
-      char *tsDefaultJSONStrType = "binary";  // todo
-      smlConvertJSONString(kv, tsDefaultJSONStrType, root);
+      smlConvertJSONString(kv, "binary", root);
       break;
     }
     case cJSON_Object: {
@@ -682,139 +484,71 @@ static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
-  int32_t ret = TSDB_CODE_SUCCESS;
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagJson(SSmlHandle *info, cJSON *tags){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
-  taosArrayClear(preLineKV);
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   int32_t tagNum = cJSON_GetArraySize(tags);
   if (unlikely(tagNum == 0)) {
     uError("SML:Tag should not be empty");
-    return TSDB_CODE_TSC_INVALID_JSON;
+    terrno = TSDB_CODE_TSC_INVALID_JSON;
+    return -1;
   }
   for (int32_t i = 0; i < tagNum; ++i) {
     cJSON *tag = cJSON_GetArrayItem(tags, i);
     if (unlikely(tag == NULL)) {
-      return TSDB_CODE_TSC_INVALID_JSON;
+      terrno = TSDB_CODE_TSC_INVALID_JSON;
+      return -1;
     }
-    //    if(unlikely(tag == cMeasure)) continue;
     size_t keyLen = strlen(tag->string);
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
       uError("OTD:Tag key length is 0 or too large than 64");
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno =  TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // add kv to SSmlKv
-    SSmlKv kv = {.key = tag->string, .keyLen = keyLen};
+    SSmlKv kv = {0};
+    kv.key = tag->string;
+    kv.keyLen = keyLen;
+
     // value
-    ret = smlParseValueFromJSON(tag, &kv);
+    int32_t ret = smlParseValueFromJSON(tag, &kv);
     if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      return ret;
-    }
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+      terrno =  ret;
+      return -1;
     }
     taosArrayPush(preLineKV, &kv);
+
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno =  TSDB_CODE_SUCCESS;
+      return -1;
+    }
+
     cnt++;
   }
-
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (unlikely(!tinfo)) {
-      return TSDB_CODE_OUT_OF_MEMORY;
+  return 0;
 }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
 
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
+static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) {
+  int32_t ret = 0;
   if(info->dataFormat){
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
   }
-
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    if(info->parseJsonByLib){
-    //      key->tags = taosMemoryMalloc(elements->tagsLen + 1);
-    //      memcpy(key->tags, elements->tags, elements->tagsLen);
-    //      key->tags[elements->tagsLen] = 0;
-    //    }
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
+  ret = smlProcessTagJson(info, tags);
+  if(ret != 0){
+    return terrno;
   }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
     return ret;
   }
+  return smlProcessChildTable(info, elements);
+}
 
 static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) {
   int32_t size = cJSON_GetArraySize(root);
@@ -998,35 +732,10 @@ static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo
     uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
@@ -1054,7 +763,6 @@ static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) {
     return TSDB_CODE_TSC_INVALID_JSON;
   }
 
-
   if (unlikely(info->lines != NULL)) {
     for (int i = 0; i < info->lineNum; i++) {
       taosArrayDestroyEx(info->lines[i].colArray, freeSSmlKv);
@@ -1202,35 +910,10 @@ static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *
       return TSDB_CODE_INVALID_TIMESTAMP;
     }
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  if (info->dataFormat) {
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
-
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }
 
 int32_t smlParseJSON(SSmlHandle *info, char *payload) {

source/client/src/clientSmlLine.c

@@ -20,15 +20,9 @@
 
 #include "clientSml.h"
 
-// comma ,
 #define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH)
-// space
 #define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH)
-// equal =
 #define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH)
-// quote "
-// #define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH)
-// SLASH
 
 #define IS_SLASH_LETTER_IN_FIELD_VALUE(sql) (*((sql)-1) == SLASH && (*(sql) == QUOTE || *(sql) == SLASH))
 
@@ -51,10 +45,10 @@
     }                                                \
   }
 
-#define BINARY_ADD_LEN 2  // "binary"   2 means " "
-#define NCHAR_ADD_LEN  3  // L"nchar"   3 means L" "
+#define BINARY_ADD_LEN (sizeof("\"\"")-1)    // "binary"   2 means length of ("")
+#define NCHAR_ADD_LEN  (sizeof("L\"\"")-1)   // L"nchar"   3 means length of (L"")
 
-uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
+uint8_t smlPrecisionConvert[] = {TSDB_TIME_PRECISION_NANO,    TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES,
                                   TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO,
                                   TSDB_TIME_PRECISION_NANO};
 
@@ -198,61 +192,10 @@ int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) {
   return TSDB_CODE_TSC_INVALID_VALUE;
 }
 
-static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
-  if (isSameCTable) {
-    return TSDB_CODE_SUCCESS;
-  }
-
-  int     cnt = 0;
+static int32_t smlProcessTagLine(SSmlHandle *info, char **sql, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES);
-        for (int i = pTableMeta->tableInfo.numOfColumns;
-             i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name,
-                         .keyLen = strlen(tag->name),
-                         .type = tag->type,
-                         .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE};
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
   taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
@@ -267,7 +210,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     while (*sql < sqlEnd) {
       if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(IS_EQUAL(*sql))) {
         keyLen = *sql - key;
@@ -283,7 +227,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen - keyLenEscaped))) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
 
     // parse value
@@ -297,7 +242,8 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         break;
       } else if (unlikely(IS_EQUAL(*sql))) {
         smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
 
       if (IS_SLASH_LETTER_IN_TAG_FIELD_KEY(*sql)) {
@@ -311,11 +257,13 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
 
     if (unlikely(valueLen == 0)) {
       smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
-      return TSDB_CODE_SML_INVALID_DATA;
+      terrno = TSDB_CODE_SML_INVALID_DATA;
+      return -1;
     }
 
     if (unlikely(valueLen - valueLenEscaped > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
     if (keyEscaped) {
@@ -338,30 +286,10 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
         .keyEscaped = keyEscaped,
         .valueEscaped = valueEscaped};
     taosArrayPush(preLineKV, &kv);
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
 
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
     }
 
     cnt++;
@@ -370,99 +298,33 @@ static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
     (*sql)++;
   }
+  return 0;
+}
 
-  void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-  if ((oneTable != NULL)) {
+static int32_t smlParseTagLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *elements) {
+  bool isSameCTable = IS_SAME_CHILD_TABLE;
+  if(isSameCTable){
     return TSDB_CODE_SUCCESS;
   }
 
-  SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen);
-  if (unlikely(!tinfo)) {
-    return TSDB_CODE_OUT_OF_MEMORY;
-  }
-  tinfo->tags = taosArrayDup(preLineKV, NULL);
-  for (size_t i = 0; i < taosArrayGetSize(preLineKV); i++) {
-    SSmlKv *kv = (SSmlKv *)taosArrayGet(preLineKV, i);
-    if (kv->keyEscaped) kv->key = NULL;
-    if (kv->valueEscaped) kv->value = NULL;
-  }
-
-  smlSetCTableName(tinfo);
-  getTableUid(info, currElement, tinfo);
+  int32_t ret = 0;
   if(info->dataFormat){
-    info->currSTableMeta->uid = tinfo->uid;
-    tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-    if (tinfo->tableDataCtx == NULL) {
-      smlDestroyTableInfo(&tinfo);
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-      return TSDB_CODE_SML_INVALID_DATA;
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
     }
   }
 
-  taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES);
-
-  return TSDB_CODE_SUCCESS;
+  ret = smlProcessTagLine(info, sql, sqlEnd);
+  if(ret != 0){
+    return terrno;
   }
 
-static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure,
-                             bool isSameCTable) {
+  return smlProcessChildTable(info, elements);
+}
+
+static int32_t smlParseColLine(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement) {
   int cnt = 0;
-  if (info->dataFormat) {
-    if (unlikely(!isSameCTable)) {
-      SSmlTableInfo **oneTable =
-          (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen);
-      if (unlikely(oneTable == NULL)) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
-      info->currTableDataCtx = (*oneTable)->tableDataCtx;
-    }
-
-    if (unlikely(!isSameMeasure)) {
-      SSmlSTableMeta **tmp =
-          (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen);
-      if (unlikely(tmp == NULL)) {
-        char *measure = currElement->measure;
-        int   measureLen = currElement->measureLen;
-        if (currElement->measureEscaped) {
-          measure = (char *)taosMemoryMalloc(currElement->measureLen);
-          memcpy(measure, currElement->measure, currElement->measureLen);
-          PROCESS_SLASH_IN_MEASUREMENT(measure, measureLen);
-          smlStrReplace(measure, measureLen);
-        }
-        STableMeta *pTableMeta = smlGetMeta(info, measure, measureLen);
-        if (currElement->measureEscaped) {
-          taosMemoryFree(measure);
-        }
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        *tmp = smlBuildSTableMeta(info->dataFormat);
-        if(*tmp == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        (*tmp)->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, currElement->measure, currElement->measureLen, tmp, POINTER_BYTES);
-
-        for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) {
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv   kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type};
-          if (tag->type == TSDB_DATA_TYPE_NCHAR) {
-            kv.length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
-          } else if (tag->type == TSDB_DATA_TYPE_BINARY || tag->type == TSDB_DATA_TYPE_GEOMETRY || tag->type == TSDB_DATA_TYPE_VARBINARY) {
-            kv.length = tag->bytes - VARSTR_HEADER_SIZE;
-          }
-          taosArrayPush((*tmp)->cols, &kv);
-        }
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxColKVs = (*tmp)->cols;
-    }
-  }
-
   while (*sql < sqlEnd) {
     if (unlikely(IS_SPACE(*sql))) {
       break;
@@ -562,47 +424,11 @@ static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLin
     }
 
     if (info->dataFormat) {
-      // cnt begin 0, add ts so + 2
-      if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
-        info->dataFormat = false;
-        info->reRun = true;
+      bool isAligned = isSmlColAligned(info, cnt, &kv);
       freeSSmlKv(&kv);
+      if(!isAligned){
         return TSDB_CODE_SUCCESS;
       }
-      // bind data
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1);
-      if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-        uDebug("smlBuildCol error, retry");
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (cnt >= taosArrayGetSize(info->maxColKVs)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxColKVs, cnt);
-      if (kv.type != maxKV->type) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        freeSSmlKv(&kv);
-        return TSDB_CODE_SUCCESS;
-      }
-
-      if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-      freeSSmlKv(&kv);
     } else {
       if (currElement->colArray == NULL) {
         currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1);
@@ -625,7 +451,6 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA;
   elements->measure = sql;
-
   // parse measure
   size_t measureLenEscaped = 0;
   while (sql < sqlEnd) {
@@ -659,20 +484,12 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     tmp++;
   }
   elements->measureTagsLen = tmp - elements->measure;
-
-  bool isSameCTable = false;
-  bool isSameMeasure = false;
-  if (IS_SAME_CHILD_TABLE) {
-    isSameCTable = true;
-    isSameMeasure = true;
-  } else if (info->dataFormat) {
-    isSameMeasure = IS_SAME_SUPER_TABLE;
-  }
+  elements->measureTag = elements->measure;
   // parse tag
   if (*sql == COMMA) sql++;
   elements->tags = sql;
 
-  int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  int ret = smlParseTagLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -687,7 +504,7 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
   JUMP_SPACE(sql, sqlEnd)
   elements->cols = sql;
 
-  ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable);
+  ret = smlParseColLine(info, &sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -718,31 +535,9 @@ int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  // add ts to
-  SSmlKv kv = {.key = tsSmlTsDefaultName,
-               .keyLen = strlen(tsSmlTsDefaultName),
-               .type = TSDB_DATA_TYPE_TIMESTAMP,
-               .i = ts,
-               .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes,
-               .keyEscaped = false,
-               .valueEscaped = false};
-  if (info->dataFormat) {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format true, ts:%" PRId64, info->id, ts);
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
 
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    uDebug("SML:0x%" PRIx64 " smlParseInfluxString format false, ts:%" PRId64, info->id, ts);
-    taosArraySet(elements->colArray, 0, &kv);
-  }
-  info->preLine = *elements;
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
 
-  return ret;
+  return smlParseEndLine(info, elements, &kvTs);
 }

source/client/src/clientSmlTelnet.c

@@ -23,8 +23,6 @@
 int32_t is_same_child_table_telnet(const void *a, const void *b) {
   SSmlLineInfo *t1 = (SSmlLineInfo *)a;
   SSmlLineInfo *t2 = (SSmlLineInfo *)b;
-  //  uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen,
-  //         t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags);
   if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
     return 1;
   return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
@@ -69,47 +67,11 @@ static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t
   }
 }
 
-static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) {
-  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
-    elements->measureTag = info->preLine.measureTag;
-    return TSDB_CODE_SUCCESS;
-  }
-
-  bool isSameMeasure = IS_SAME_SUPER_TABLE;
-
-  int     cnt = 0;
+static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
   SArray *preLineKV = info->preLineTagKV;
-  if (info->dataFormat) {
-    if (!isSameMeasure) {
-      SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
-      SSmlSTableMeta *sMeta = NULL;
-      if (unlikely(tmp == NULL)) {
-        STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen);
-        if (pTableMeta == NULL) {
-          info->dataFormat = false;
-          info->reRun = true;
-          return TSDB_CODE_SUCCESS;
-        }
-        sMeta = smlBuildSTableMeta(info->dataFormat);
-        if(sMeta == NULL){
-          taosMemoryFreeClear(pTableMeta);
-          return TSDB_CODE_OUT_OF_MEMORY;
-        }
-        sMeta->tableMeta = pTableMeta;
-        taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES);
-        for(int i = pTableMeta->tableInfo.numOfColumns; i < pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; i++){
-          SSchema *tag = pTableMeta->schema + i;
-          SSmlKv kv = {.key = tag->name, .keyLen = strlen(tag->name), .type = tag->type, .length = (tag->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE };
-          taosArrayPush(sMeta->tags, &kv);
-        }
-        tmp = &sMeta;
-      }
-      info->currSTableMeta = (*tmp)->tableMeta;
-      info->maxTagKVs = (*tmp)->tags;
-    }
-  }
+  taosArrayClearEx(preLineKV, freeSSmlKv);
+  int     cnt = 0;
 
-  taosArrayClear(preLineKV);
   const char *sql = data;
   while (sql < sqlEnd) {
     JUMP_SPACE(sql, sqlEnd)
@@ -121,8 +83,9 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     // parse key
     while (sql < sqlEnd) {
       if (unlikely(*sql == SPACE)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       if (unlikely(*sql == EQUAL)) {
         keyLen = sql - key;
@@ -133,13 +96,10 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
     }
 
     if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
-      smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key);
-      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
+      terrno = TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
+      return -1;
     }
-    //    if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) {
-    //      smlBuildInvalidDataMsg(msg, "dumplicate key", key);
-    //      return TSDB_CODE_TSC_DUP_NAMES;
-    //    }
 
     // parse value
     const char *value = sql;
@@ -150,89 +110,69 @@ static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SS
         break;
       }
       if (unlikely(*sql == EQUAL)) {
-        smlBuildInvalidDataMsg(msg, "invalid data", sql);
-        return TSDB_CODE_SML_INVALID_DATA;
+        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
+        terrno = TSDB_CODE_SML_INVALID_DATA;
+        return -1;
       }
       sql++;
     }
     valueLen = sql - value;
 
     if (unlikely(valueLen == 0)) {
-      smlBuildInvalidDataMsg(msg, "invalid value", value);
-      return TSDB_CODE_TSC_INVALID_VALUE;
+      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
+      terrno = TSDB_CODE_TSC_INVALID_VALUE;
+      return -1;
     }
 
     if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
-      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      terrno = TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
+      return -1;
     }
 
-    SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen};
-
-    if (info->dataFormat) {
-      if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(cnt >= taosArrayGetSize(info->maxTagKVs))) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      SSmlKv *maxKV = (SSmlKv *)taosArrayGet(info->maxTagKVs, cnt);
-      if (unlikely(!IS_SAME_KEY)) {
-        info->dataFormat = false;
-        info->reRun = true;
-        return TSDB_CODE_SUCCESS;
-      }
-      if (unlikely(kv.length > maxKV->length)) {
-        maxKV->length = kv.length;
-        info->needModifySchema = true;
-      }
-    }
+    SSmlKv kv = {.key = key,
+        .keyLen = keyLen,
+        .type = TSDB_DATA_TYPE_NCHAR,
+        .value = value,
+        .length = valueLen,
+        .keyEscaped = false,
+        .valueEscaped = false};
     taosArrayPush(preLineKV, &kv);
+    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
+      terrno = TSDB_CODE_SUCCESS;
+      return -1;
+    }
     cnt++;
   }
-
-  elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
-  memcpy(elements->measureTag, elements->measure, elements->measureLen);
-  memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen);
-  elements->measureTagsLen = elements->measureLen + elements->tagsLen;
-
-  SSmlTableInfo **tmp =
-      (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen);
-  SSmlTableInfo *tinfo = NULL;
-  if (unlikely(tmp == NULL)) {
-    tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen);
-    if (!tinfo) {
-      return TSDB_CODE_OUT_OF_MEMORY;
-    }
-    tinfo->tags = taosArrayDup(preLineKV, NULL);
-
-    smlSetCTableName(tinfo);
-    getTableUid(info, elements, tinfo);
-    if (info->dataFormat) {
-      info->currSTableMeta->uid = tinfo->uid;
-      tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta);
-      if (tinfo->tableDataCtx == NULL) {
-        smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL);
-        smlDestroyTableInfo(&tinfo);
-        return TSDB_CODE_SML_INVALID_DATA;
-      }
+  return 0;
 }
 
-    //    SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo));
-    //    *key = *elements;
-    //    tinfo->key = key;
-    taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo,
-                POINTER_BYTES);
-    tmp = &tinfo;
-  }
-  if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx;
-
+static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
+  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
+    elements->measureTag = info->preLine.measureTag;
     return TSDB_CODE_SUCCESS;
   }
 
+  int32_t ret = 0;
+  if(info->dataFormat){
+    ret = smlProcessSuperTable(info, elements);
+    if(ret != 0){
+      return terrno;
+    }
+  }
+
+  ret = smlProcessTagTelnet(info, data, sqlEnd);
+  if(ret != 0){
+    return terrno;
+  }
+
+  ret = smlJoinMeasureTag(elements);
+  if(ret != 0){
+    return ret;
+  }
+
+  return smlProcessChildTable(info, elements);
+}
+
 // format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
 int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) {
   if (!sql) return TSDB_CODE_SML_INVALID_DATA;
@@ -251,7 +191,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SML_INVALID_DATA;
   }
 
-  bool needConverTime = false;  // get TS before parse tag(get meta), so need conver time
+  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
   if (info->dataFormat && info->currSTableMeta == NULL) {
     needConverTime = true;
   }
@@ -260,11 +200,6 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql);
     return TSDB_CODE_INVALID_TIMESTAMP;
   }
-  SSmlKv kvTs = {.key = tsSmlTsDefaultName,
-                 .keyLen = strlen(tsSmlTsDefaultName),
-                 .type = TSDB_DATA_TYPE_TIMESTAMP,
-                 .i = ts,
-                 .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes};
 
   // parse value
   smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
@@ -287,7 +222,7 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_TSC_INVALID_VALUE;
   }
 
-  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf);
+  int ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
   if (unlikely(ret != TSDB_CODE_SUCCESS)) {
     return ret;
   }
@@ -296,30 +231,11 @@ int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLine
     return TSDB_CODE_SUCCESS;
   }
 
-  if (info->dataFormat && info->currSTableMeta != NULL) {
+  SSmlKv kvTs = {0};
+  smlBuildTsKv(&kvTs, ts);
   if (needConverTime) {
     kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
   }
-    ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0);
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1);
-    }
-    if (ret == TSDB_CODE_SUCCESS) {
-      ret = smlBuildRow(info->currTableDataCtx);
-    }
-    clearColValArraySml(info->currTableDataCtx->pValues);
-    if (unlikely(ret != TSDB_CODE_SUCCESS)) {
-      smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL);
-      return ret;
-    }
-  } else {
-    if (elements->colArray == NULL) {
-      elements->colArray = taosArrayInit(16, sizeof(SSmlKv));
-    }
-    taosArrayPush(elements->colArray, &kvTs);
-    taosArrayPush(elements->colArray, &kv);
-  }
-  info->preLine = *elements;
 
-  return TSDB_CODE_SUCCESS;
+  return smlParseEndTelnetJson(info, elements, &kvTs, &kv);
 }

source/common/CMakeLists.txt

@@ -60,8 +60,8 @@ if(${BUILD_S3})
         find_library(S3_LIBRARY s3)
         find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
         find_library(XML2_LIBRARY xml2)
-        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
-        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH)
+        find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
+        find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH)
         target_link_libraries(
                 common
 

source/common/src/cos.c

@@ -1,6 +1,8 @@
 #define ALLOW_FORBID_FUNC
 
 #include "cos.h"
+#include "cos_cp.h"
+#include "tdef.h"
 
 extern char   tsS3Endpoint[];
 extern char   tsS3AccessKeyId[];
@@ -66,11 +68,12 @@ static int should_retry() {
   return 0;
 }
 
-static void s3PrintError(const char *func, S3Status status, char error_details[]) {
+static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status,
+                         char error_details[]) {
   if (status < S3StatusErrorAccessDenied) {
-    uError("%s: %s", __func__, S3_get_status_name(status));
+    uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status));
   } else {
-    uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+    uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details);
   }
 }
 
@@ -230,9 +233,9 @@ typedef struct put_object_callback_data {
   int         noStatus;
 } put_object_callback_data;
 
-#define MULTIPART_CHUNK_SIZE (768 << 20)  // multipart is 768M
+#define MULTIPART_CHUNK_SIZE (64 << 20)  // multipart is 768M
 
-typedef struct UploadManager {
+typedef struct {
   char     err_msg[512];
   S3Status status;
   uint64_t content_length;
@@ -307,6 +310,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat
 S3Status initial_multipart_callback(const char *upload_id, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   manager->upload_id = strdup(upload_id);
+  manager->status = S3StatusOK;
   return S3StatusOK;
 }
 
@@ -321,6 +325,17 @@ S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properti
   return S3StatusOK;
 }
 
+S3Status MultipartResponseProperiesCallbackWithCp(const S3ResponseProperties *properties, void *callbackData) {
+  responsePropertiesCallbackNull(properties, callbackData);
+
+  MultipartPartData *data = (MultipartPartData *)callbackData;
+  int                seq = data->seq;
+  const char        *etag = properties->eTag;
+  data->manager->etags[seq - 1] = strdup(etag);
+  // data->manager->next_etags_pos = seq;
+  return S3StatusOK;
+}
+
 static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) {
   UploadManager *manager = (UploadManager *)callbackData;
   int            ret = 0;
@@ -437,86 +452,46 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan
       // printListMultipartHeader(data.allDetails);
     }
   } else {
-    s3PrintError(__func__, data.status, data.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
     return -1;
   }
 
   return 0;
 }
 */
-int32_t s3PutObjectFromFile2(const char *file, const char *object) {
+
+static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char const *object_name, int64_t size,
+                                         S3PutProperties *put_prop, put_object_callback_data *data) {
   int32_t            code = 0;
-  const char *key = object;
-  // const char              *uploadId = 0;
-  const char              *filename = 0;
-  uint64_t                 contentLength = 0;
-  const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
-  const char              *contentDispositionFilename = 0, *contentEncoding = 0;
-  int64_t                  expires = -1;
-  S3CannedAcl              cannedAcl = S3CannedAclPrivate;
-  int                      metaPropertiesCount = 0;
-  S3NameValue              metaProperties[S3_MAX_METADATA_COUNT];
-  char                     useServerSideEncryption = 0;
-  put_object_callback_data data = {0};
-  // int                      noStatus = 0;
-
-  // data.infile = 0;
-  // data.gb = 0;
-  // data.infileFD = NULL;
-  // data.noStatus = noStatus;
-
-  if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
-    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
-    code = TAOS_SYSTEM_ERROR(errno);
-    return code;
-  }
-
-  if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
-    uError("ERROR: %s Failed to open file %s: ", __func__, file);
-    code = TAOS_SYSTEM_ERROR(errno);
-    return code;
-  }
-
-  data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
-      contentLength;
-
-  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
-                                   0, awsRegionG};
-
-  S3PutProperties putProperties = {contentType,     md5,
-                                   cacheControl,    contentDispositionFilename,
-                                   contentEncoding, expires,
-                                   cannedAcl,       metaPropertiesCount,
-                                   metaProperties,  useServerSideEncryption};
-
-  if (contentLength <= MULTIPART_CHUNK_SIZE) {
   S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
                                          &putObjectDataCallback};
 
   do {
-      S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data);
-    } while (S3_status_is_retryable(data.status) && should_retry());
+    S3_put_object(bucket_context, object_name, size, put_prop, 0, 0, &putObjectHandler, data);
+  } while (S3_status_is_retryable(data->status) && should_retry());
 
-    if (data.status != S3StatusOK) {
-      s3PrintError(__func__, data.status, data.err_msg);
+  if (data->status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
     code = TAOS_SYSTEM_ERROR(EIO);
-    } else if (data.contentLength) {
-      uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__,
-             (unsigned long long)data.contentLength);
+  } else if (data->contentLength) {
+    uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength);
     code = TAOS_SYSTEM_ERROR(EIO);
   }
-  } else {
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithoutCp(S3BucketContext *bucket_context, char const *object_name,
+                                            int64_t contentLength, S3PutProperties *put_prop,
+                                            put_object_callback_data *data) {
+  int32_t       code = 0;
   uint64_t      totalContentLength = contentLength;
   uint64_t      todoContentLength = contentLength;
-    UploadManager manager;
-    manager.upload_id = 0;
-    manager.gb = 0;
+  UploadManager manager = {0};
 
-    // div round up
-    int       seq;
-    uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 7;
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
   int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
-    const int max_part_num = 1000;
+  const int max_part_num = 10000;
   if (totalSeq > max_part_num) {
     chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
     totalSeq = (contentLength + chunk_size - 1) / chunk_size;
@@ -535,39 +510,25 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   S3MultipartCommitHandler commit_handler = {
       {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
 
-    manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq);
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
   manager.next_etags_pos = 0;
-    /*
-    if (uploadId) {
-      manager.upload_id = strdup(uploadId);
-      manager.remaining = contentLength;
-      if (!try_get_parts_info(tsS3BucketName, key, &manager)) {
-        fseek(data.infile, -(manager.remaining), 2);
-        taosLSeekFile(data.infileFD, -(manager.remaining), SEEK_END);
-        contentLength = manager.remaining;
-        goto upload;
-      } else {
-        goto clean;
-      }
-    }
-    */
   do {
-      S3_initiate_multipart(&bucketContext, key, 0, &handler, 0, timeoutMsG, &manager);
+    S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
   } while (S3_status_is_retryable(manager.status) && should_retry());
 
   if (manager.upload_id == 0 || manager.status != S3StatusOK) {
-      s3PrintError(__func__, manager.status, manager.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
     code = TAOS_SYSTEM_ERROR(EIO);
     goto clean;
   }
 
 upload:
   todoContentLength -= chunk_size * manager.next_etags_pos;
-    for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
+  for (int seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) {
     partData.manager = &manager;
     partData.seq = seq;
     if (partData.put_object_data.gb == NULL) {
-        partData.put_object_data = data;
+      partData.put_object_data = *data;
     }
     partContentLength = ((contentLength > chunk_size) ? chunk_size : contentLength);
     // printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength);
@@ -575,13 +536,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
     partData.put_object_data.originalContentLength = partContentLength;
     partData.put_object_data.totalContentLength = todoContentLength;
     partData.put_object_data.totalOriginalContentLength = totalContentLength;
-      putProperties.md5 = 0;
+    put_prop->md5 = 0;
     do {
-        S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id,
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
                      partContentLength, 0, timeoutMsG, &partData);
     } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
     if (partData.put_object_data.status != S3StatusOK) {
-        s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
       code = TAOS_SYSTEM_ERROR(EIO);
       goto clean;
     }
@@ -595,6 +556,10 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   char buf[256];
   int  n;
   for (i = 0; i < totalSeq; i++) {
+    if (!manager.etags[i]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
     n = snprintf(buf, sizeof(buf),
                  "<Part><PartNumber>%d</PartNumber>"
                  "<ETag>%s</ETag></Part>",
@@ -605,11 +570,11 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   manager.remaining = size;
 
   do {
-      S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, 0,
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
                                  timeoutMsG, &manager);
   } while (S3_status_is_retryable(manager.status) && should_retry());
   if (manager.status != S3StatusOK) {
-      s3PrintError(__func__, manager.status, manager.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
     code = TAOS_SYSTEM_ERROR(EIO);
     goto clean;
   }
@@ -621,16 +586,247 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   for (i = 0; i < manager.next_etags_pos; i++) {
     taosMemoryFree(manager.etags[i]);
   }
+  growbuffer_destroy(manager.gb);
+  taosMemoryFree(manager.etags);
+
+  return code;
+}
+
+static int32_t s3PutObjectFromFileWithCp(S3BucketContext *bucket_context, const char *file, int32_t lmtime,
+                                         char const *object_name, int64_t contentLength, S3PutProperties *put_prop,
+                                         put_object_callback_data *data) {
+  int32_t code = 0;
+
+  uint64_t totalContentLength = contentLength;
+  // uint64_t      todoContentLength = contentLength;
+  UploadManager manager = {0};
+
+  uint64_t  chunk_size = MULTIPART_CHUNK_SIZE >> 3;
+  int       totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  const int max_part_num = 10000;
+  if (totalSeq > max_part_num) {
+    chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num;
+    totalSeq = (contentLength + chunk_size - 1) / chunk_size;
+  }
+
+  bool need_init_upload = true;
+  char file_cp_path[TSDB_FILENAME_LEN];
+  snprintf(file_cp_path, TSDB_FILENAME_LEN, "%s.cp", file);
+
+  SCheckpoint cp = {0};
+  cp.parts = taosMemoryCalloc(max_part_num, sizeof(SCheckpointPart));
+
+  if (taosCheckExistFile(file_cp_path)) {
+    if (!cos_cp_load(file_cp_path, &cp) && cos_cp_is_valid_upload(&cp, contentLength, lmtime)) {
+      manager.upload_id = strdup(cp.upload_id);
+      need_init_upload = false;
+    } else {
+      cos_cp_remove(file_cp_path);
+    }
+  }
+
+  if (need_init_upload) {
+    S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback},
+                                         &initial_multipart_callback};
+    do {
+      S3_initiate_multipart(bucket_context, object_name, 0, &handler, 0, timeoutMsG, &manager);
+    } while (S3_status_is_retryable(manager.status) && should_retry());
+
+    if (manager.upload_id == 0 || manager.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_build_upload(&cp, file, contentLength, lmtime, manager.upload_id, chunk_size);
+  }
+
+  if (cos_cp_open(file_cp_path, &cp)) {
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  int     part_num = 0;
+  int64_t consume_bytes = 0;
+  // SCheckpointPart *parts = taosMemoryCalloc(cp.part_num, sizeof(SCheckpointPart));
+  // cos_cp_get_undo_parts(&cp, &part_num, parts, &consume_bytes);
+
+  MultipartPartData partData;
+  memset(&partData, 0, sizeof(MultipartPartData));
+  int partContentLength = 0;
+
+  S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallbackWithCp, &responseCompleteCallback},
+                                         &putObjectDataCallback};
+
+  S3MultipartCommitHandler commit_handler = {
+      {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0};
+
+  manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *));
+  manager.next_etags_pos = 0;
+
+upload:
+  // todoContentLength -= chunk_size * manager.next_etags_pos;
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (cp.parts[i].completed) {
+      continue;
+    }
+
+    if (i > 0 && cp.parts[i - 1].completed) {
+      if (taosLSeekFile(data->infileFD, cp.parts[i].offset, SEEK_SET) < 0) {
+        code = TAOS_SYSTEM_ERROR(errno);
+        goto clean;
+      }
+    }
+
+    int seq = cp.parts[i].index + 1;
+
+    partData.manager = &manager;
+    partData.seq = seq;
+    if (partData.put_object_data.gb == NULL) {
+      partData.put_object_data = *data;
+    }
+
+    partContentLength = cp.parts[i].size;
+    partData.put_object_data.contentLength = partContentLength;
+    partData.put_object_data.originalContentLength = partContentLength;
+    // partData.put_object_data.totalContentLength = todoContentLength;
+    partData.put_object_data.totalOriginalContentLength = totalContentLength;
+    put_prop->md5 = 0;
+    do {
+      S3_upload_part(bucket_context, object_name, put_prop, &putObjectHandler, seq, manager.upload_id,
+                     partContentLength, 0, timeoutMsG, &partData);
+    } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry());
+    if (partData.put_object_data.status != S3StatusOK) {
+      s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg);
+      code = TAOS_SYSTEM_ERROR(EIO);
+
+      //(void)cos_cp_dump(&cp);
+      goto clean;
+    }
+
+    if (!manager.etags[seq - 1]) {
+      code = TAOS_SYSTEM_ERROR(EIO);
+      goto clean;
+    }
+
+    cos_cp_update(&cp, cp.parts[seq - 1].index, manager.etags[seq - 1], 0);
+    (void)cos_cp_dump(&cp);
+
+    contentLength -= chunk_size;
+    // todoContentLength -= chunk_size;
+  }
+
+  cos_cp_close(cp.thefile);
+  cp.thefile = 0;
+
+  int size = 0;
+  size += growbuffer_append(&(manager.gb), "<CompleteMultipartUpload>", strlen("<CompleteMultipartUpload>"));
+  char buf[256];
+  int  n;
+  for (int i = 0; i < cp.part_num; ++i) {
+    n = snprintf(buf, sizeof(buf),
+                 "<Part><PartNumber>%d</PartNumber>"
+                 "<ETag>%s</ETag></Part>",
+                 // i + 1, manager.etags[i]);
+                 cp.parts[i].index + 1, cp.parts[i].etag);
+    size += growbuffer_append(&(manager.gb), buf, n);
+  }
+  size += growbuffer_append(&(manager.gb), "</CompleteMultipartUpload>", strlen("</CompleteMultipartUpload>"));
+  manager.remaining = size;
+
+  do {
+    S3_complete_multipart_upload(bucket_context, object_name, &commit_handler, manager.upload_id, manager.remaining, 0,
+                                 timeoutMsG, &manager);
+  } while (S3_status_is_retryable(manager.status) && should_retry());
+  if (manager.status != S3StatusOK) {
+    s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg);
+    code = TAOS_SYSTEM_ERROR(EIO);
+    goto clean;
+  }
+
+  cos_cp_remove(file_cp_path);
+
+clean:
+  /*
+  if (parts) {
+    taosMemoryFree(parts);
+  }
+  */
+  if (cp.thefile) {
+    cos_cp_close(cp.thefile);
+  }
+  if (cp.parts) {
+    taosMemoryFree(cp.parts);
+  }
+
+  if (manager.upload_id) {
+    taosMemoryFree(manager.upload_id);
+  }
+  for (int i = 0; i < cp.part_num; ++i) {
+    if (manager.etags[i]) {
+      taosMemoryFree(manager.etags[i]);
+    }
+  }
+  taosMemoryFree(manager.etags);
+  growbuffer_destroy(manager.gb);
+
+  return code;
+}
+
+int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t withcp) {
+  int32_t                  code = 0;
+  int32_t                  lmtime = 0;
+  const char              *filename = 0;
+  uint64_t                 contentLength = 0;
+  const char              *cacheControl = 0, *contentType = 0, *md5 = 0;
+  const char              *contentDispositionFilename = 0, *contentEncoding = 0;
+  int64_t                  expires = -1;
+  S3CannedAcl              cannedAcl = S3CannedAclPrivate;
+  int                      metaPropertiesCount = 0;
+  S3NameValue              metaProperties[S3_MAX_METADATA_COUNT];
+  char                     useServerSideEncryption = 0;
+  put_object_callback_data data = {0};
+
+  if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to stat file %s: ", __func__, file);
+    return code;
+  }
+
+  if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open file %s: ", __func__, file);
+    return code;
+  }
+
+  data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
+      contentLength;
+
+  S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
+                                   0, awsRegionG};
+
+  S3PutProperties putProperties = {contentType,     md5,
+                                   cacheControl,    contentDispositionFilename,
+                                   contentEncoding, expires,
+                                   cannedAcl,       metaPropertiesCount,
+                                   metaProperties,  useServerSideEncryption};
+
+  if (contentLength <= MULTIPART_CHUNK_SIZE) {
+    code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
+  } else {
+    if (withcp) {
+      code = s3PutObjectFromFileWithCp(&bucketContext, file, lmtime, object_name, contentLength, &putProperties, &data);
+    } else {
+      code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
+    }
+  }
+
   if (data.infileFD) {
     taosCloseFile(&data.infileFD);
   } else if (data.gb) {
     growbuffer_destroy(data.gb);
   }
 
-    growbuffer_destroy(manager.gb);
-    taosMemoryFree(manager.etags);
-  }
-
   return code;
 }
 
@@ -722,7 +918,7 @@ static SArray *getListByPrefix(const char *prefix) {
       return data.objectArray;
     }
   } else {
-    s3PrintError(__func__, data.status, data.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
   }
 
   taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
@@ -741,7 +937,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
     } while (S3_status_is_retryable(cbd.status) && should_retry());
 
     if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-      s3PrintError(__func__, cbd.status, cbd.err_msg);
+      s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
     }
   }
 }
@@ -884,7 +1080,9 @@ long s3Size(const char *object_name) {
   } while (S3_status_is_retryable(cbd.status) && should_retry());
 
   if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
-    uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg);
+    s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
+
+    return -1;
   }
 
   size = cbd.content_length;
@@ -974,7 +1172,7 @@ int32_t s3PutObjectFromFile(const char *file_str, const char *object_str) {
   return code;
 }
 
-int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
+int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str, int8_t withcp) {
   int32_t                     code = 0;
   cos_pool_t                 *p = NULL;
   int                         is_cname = 0;
@@ -986,6 +1184,7 @@ int32_t s3PutObjectFromFile2(const char *file_str, const char *object_str) {
   cos_table_t                *headers = NULL;
   cos_resumable_clt_params_t *clt_params = NULL;
 
+  (void)withcp;
   cos_pool_create(&p, NULL);
   options = cos_request_options_create(p);
   s3InitRequestOptions(options, is_cname);
@@ -1335,7 +1534,7 @@ long s3Size(const char *object_name) {
 int32_t s3Init() { return 0; }
 void    s3CleanUp() {}
 int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
-int32_t s3PutObjectFromFile2(const char *file, const char *object) { return 0; }
+int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
 void    s3DeleteObjectsByPrefix(const char *prefix) {}
 void    s3DeleteObjects(const char *object_name[], int nobject) {}
 bool    s3Exists(const char *object_name) { return false; }
@@ -1346,5 +1545,6 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
 void    s3EvictCache(const char *path, long object_size) {}
 long    s3Size(const char *object_name) { return 0; }
 int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; }
+int32_t s3GetObjectToFile(const char *object_name, char *fileName) { return 0; }
 
 #endif

source/common/src/cos_cp.c

@@ -0,0 +1,417 @@
+#define ALLOW_FORBID_FUNC
+
+#include "cos_cp.h"
+#include "cJSON.h"
+#include "tutil.h"
+
+int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(cp_path, TD_FILE_WRITE | TD_FILE_CREATE /* | TD_FILE_TRUNC*/ | TD_FILE_WRITE_THROUGH);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, cp_path);
+    return code;
+  }
+
+  checkpoint->thefile = fd;
+
+  return code;
+}
+
+void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); }
+void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); }
+
+static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) {
+  int32_t      code = 0;
+  cJSON const* item2 = NULL;
+
+  cJSON* json = cJSON_Parse(cp_body);
+  if (NULL == json) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+
+  cJSON const* item = cJSON_GetObjectItem(json, "ver");
+  if (!cJSON_IsNumber(item) || item->valuedouble != 1) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json ver: %f", __func__, item->valuedouble);
+    goto _exit;
+  }
+
+  item = cJSON_GetObjectItem(json, "type");
+  if (!cJSON_IsNumber(item)) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to parse json", __func__);
+    goto _exit;
+  }
+  cp->cp_type = item->valuedouble;
+
+  item = cJSON_GetObjectItem(json, "md5");
+  if (cJSON_IsString(item)) {
+    memcpy(cp->md5, item->valuestring, strlen(item->valuestring));
+  }
+
+  item = cJSON_GetObjectItem(json, "upload_id");
+  if (cJSON_IsString(item)) {
+    strncpy(cp->upload_id, item->valuestring, 128);
+  }
+
+  item2 = cJSON_GetObjectItem(json, "file");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->file_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "lastmodified");
+    if (cJSON_IsNumber(item)) {
+      cp->file_last_modified = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "path");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_path, item->valuestring, TSDB_FILENAME_LEN);
+    }
+
+    item = cJSON_GetObjectItem(item2, "file_md5");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->file_md5, item->valuestring, 64);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "object");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "object_size");
+    if (cJSON_IsNumber(item)) {
+      cp->object_size = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_name");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_name, item->valuestring, 128);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_last_modified");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_last_modified, item->valuestring, 64);
+    }
+
+    item = cJSON_GetObjectItem(item2, "object_etag");
+    if (cJSON_IsString(item)) {
+      strncpy(cp->object_etag, item->valuestring, 128);
+    }
+  }
+
+  item2 = cJSON_GetObjectItem(json, "cpparts");
+  if (cJSON_IsObject(item2)) {
+    item = cJSON_GetObjectItem(item2, "number");
+    if (cJSON_IsNumber(item)) {
+      cp->part_num = item->valuedouble;
+    }
+
+    item = cJSON_GetObjectItem(item2, "size");
+    if (cJSON_IsNumber(item)) {
+      cp->part_size = item->valuedouble;
+    }
+
+    item2 = cJSON_GetObjectItem(item2, "parts");
+    if (cJSON_IsArray(item2) && cp->part_num > 0) {
+      cJSON_ArrayForEach(item, item2) {
+        cJSON const* item3 = cJSON_GetObjectItem(item, "index");
+        int32_t      index = 0;
+        if (cJSON_IsNumber(item3)) {
+          index = item3->valuedouble;
+          cp->parts[index].index = index;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "offset");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].offset = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "size");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].size = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "completed");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].completed = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "crc64");
+        if (cJSON_IsNumber(item3)) {
+          cp->parts[index].crc64 = item3->valuedouble;
+        }
+
+        item3 = cJSON_GetObjectItem(item, "etag");
+        if (cJSON_IsString(item3)) {
+          strncpy(cp->parts[index].etag, item3->valuestring, 128);
+        }
+      }
+    }
+  }
+
+_exit:
+  if (json) cJSON_Delete(json);
+  if (cp_body) taosMemoryFree(cp_body);
+
+  return code;
+}
+
+int32_t cos_cp_load(char const* filepath, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  TdFilePtr fd = taosOpenFile(filepath, TD_FILE_READ);
+  if (!fd) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to open %s", __func__, filepath);
+    goto _exit;
+  }
+
+  int64_t size = -1;
+  code = taosStatFile(filepath, &size, NULL, NULL);
+  if (code) {
+    uError("ERROR: %s Failed to stat %s", __func__, filepath);
+    goto _exit;
+  }
+
+  char* cp_body = taosMemoryMalloc(size + 1);
+
+  int64_t n = taosReadFile(fd, cp_body, size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    uError("ERROR: %s Failed to read %s", __func__, filepath);
+    goto _exit;
+  } else if (n != size) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    uError("ERROR: %s Failed to read %s %" PRId64 "/%" PRId64, __func__, filepath, n, size);
+    goto _exit;
+  }
+  taosCloseFile(&fd);
+  cp_body[size] = '\0';
+
+  return cos_cp_parse_body(cp_body, checkpoint);
+
+_exit:
+  if (fd) {
+    taosCloseFile(&fd);
+  }
+  if (cp_body) {
+    taosMemoryFree(cp_body);
+  }
+
+  return code;
+}
+
+static int32_t cos_cp_save_json(cJSON const* json, SCheckpoint* checkpoint) {
+  int32_t code = 0;
+
+  char* data = cJSON_PrintUnformatted(json);
+  if (NULL == data) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  TdFilePtr fp = checkpoint->thefile;
+  if (taosFtruncateFile(fp, 0) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosLSeekFile(fp, 0, SEEK_SET) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+  if (taosWriteFile(fp, data, strlen(data)) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+  if (taosFsyncFile(fp) < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
+  }
+
+_exit:
+  taosMemoryFree(data);
+  return code;
+}
+
+int32_t cos_cp_dump(SCheckpoint* cp) {
+  int32_t code = 0;
+  int32_t lino = 0;
+
+  cJSON* ojson = NULL;
+  cJSON* json = cJSON_CreateObject();
+  if (!json) return TSDB_CODE_OUT_OF_MEMORY;
+
+  if (NULL == cJSON_AddNumberToObject(json, "ver", 1)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddNumberToObject(json, "type", cp->cp_type)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "md5", cp->md5)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (NULL == cJSON_AddStringToObject(json, "upload_id", cp->upload_id)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  if (COS_CP_TYPE_UPLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "file");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->file_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "lastmodified", cp->file_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "path", cp->file_path)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "file_md5", cp->file_md5)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  } else if (COS_CP_TYPE_DOWNLOAD == cp->cp_type) {
+    ojson = cJSON_AddObjectToObject(json, "object");
+    if (!ojson) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(ojson, "object_size", cp->object_size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_name", cp->object_name)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_last_modified", cp->object_last_modified)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(ojson, "object_etag", cp->object_etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  ojson = cJSON_AddObjectToObject(json, "cpparts");
+  if (!ojson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "number", cp->part_num)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  if (NULL == cJSON_AddNumberToObject(ojson, "size", cp->part_size)) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+
+  cJSON* ajson = cJSON_AddArrayToObject(ojson, "parts");
+  if (!ajson) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    TSDB_CHECK_CODE(code, lino, _exit);
+  }
+  for (int i = 0; i < cp->part_num; ++i) {
+    cJSON* item = cJSON_CreateObject();
+    if (!item) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    cJSON_AddItemToArray(ajson, item);
+
+    if (NULL == cJSON_AddNumberToObject(item, "index", cp->parts[i].index)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "offset", cp->parts[i].offset)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "size", cp->parts[i].size)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "completed", cp->parts[i].completed)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddNumberToObject(item, "crc64", cp->parts[i].crc64)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+    if (NULL == cJSON_AddStringToObject(item, "etag", cp->parts[i].etag)) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      TSDB_CHECK_CODE(code, lino, _exit);
+    }
+  }
+
+  code = cos_cp_save_json(json, cp);
+  TSDB_CHECK_CODE(code, lino, _exit);
+
+_exit:
+  if (code) {
+    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
+  }
+  cJSON_Delete(json);
+  return code;
+}
+
+void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPart* parts, int64_t* consume_bytes) {}
+
+void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) {
+  checkpoint->parts[part_index].completed = 1;
+  strncpy(checkpoint->parts[part_index].etag, etag, 128);
+  checkpoint->parts[part_index].crc64 = crc64;
+}
+
+void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t size, int32_t mtime,
+                         char const* upload_id, int64_t part_size) {
+  int i = 0;
+
+  checkpoint->cp_type = COS_CP_TYPE_UPLOAD;
+  strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN);
+
+  checkpoint->file_size = size;
+  checkpoint->file_last_modified = mtime;
+  strncpy(checkpoint->upload_id, upload_id, 128);
+
+  checkpoint->part_size = part_size;
+  for (; i * part_size < size; i++) {
+    checkpoint->parts[i].index = i;
+    checkpoint->parts[i].offset = i * part_size;
+    checkpoint->parts[i].size = TMIN(part_size, (size - i * part_size));
+    checkpoint->parts[i].completed = 0;
+    checkpoint->parts[i].etag[0] = '\0';
+  }
+  checkpoint->part_num = i;
+}
+
+static bool cos_cp_verify_md5(SCheckpoint* cp) { return true; }
+
+bool cos_cp_is_valid_upload(SCheckpoint* checkpoint, int64_t size, int32_t mtime) {
+  if (cos_cp_verify_md5(checkpoint) && checkpoint->file_size == size && checkpoint->file_last_modified == mtime) {
+    return true;
+  }
+
+  return false;
+}

source/common/src/systable.c

@@ -140,7 +140,7 @@ static const SSysDbTableSchema userStbsSchema[] = {
     {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
-    {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "table_comment", .bytes = TSDB_TB_COMMENT_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "watermark", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "max_delay", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "rollup", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@@ -167,7 +167,7 @@ static const SSysDbTableSchema streamTaskSchema[] = {
     {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
     {.name = "level", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "status", .bytes = 15 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
-    {.name = "stage", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
     {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 //    {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
     {.name = "info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@@ -334,6 +334,20 @@ static const SSysDbTableSchema userViewsSchema[] = {
 //    {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
 };
 
+static const SSysDbTableSchema userCompactsSchema[] = {
+    {.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+};
+
+static const SSysDbTableSchema userCompactsDetailSchema[] = {
+    {.name = "compact_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "number_fileset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
+    {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
+};
 
 static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
@@ -362,6 +376,8 @@ static const SSysTableMeta infosMeta[] = {
     {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
     {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
     {TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
+    {TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
+    {TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
 };
 
 static const SSysDbTableSchema connectionsSchema[] = {

source/common/src/tdataformat.c

@@ -1248,6 +1248,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
     n += tPutCStr(p ? p + n : p, pTagVal->pKey);
   } else {
     n += tPutI16v(p ? p + n : p, pTagVal->cid);
+    ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
   }
 
   // type
@@ -3590,9 +3591,9 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_
     tColDataCalcSMAUInt,           // TSDB_DATA_TYPE_UINT
     tColDataCalcSMAUBigInt,        // TSDB_DATA_TYPE_UBIGINT
     tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_JSON
-    NULL,                          // TSDB_DATA_TYPE_VARBINARY
-    NULL,                          // TSDB_DATA_TYPE_DECIMAL
-    NULL,                          // TSDB_DATA_TYPE_BLOB
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_VARBINARY
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_DECIMAL
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_BLOB
     NULL,                          // TSDB_DATA_TYPE_MEDIUMBLOB
     tColDataCalcSMAVarType         // TSDB_DATA_TYPE_GEOMETRY
 };

source/common/src/tglobal.c

@@ -248,6 +248,7 @@ int32_t tsTtlBatchDropNum = 10000;   // number of tables dropped per batch
 
 // internal
 int32_t tsTransPullupInterval = 2;
+int32_t tsCompactPullupInterval = 10;
 int32_t tsMqRebalanceInterval = 2;
 int32_t tsStreamCheckpointInterval = 60;
 float   tsSinkDataRate = 2.0;
@@ -279,7 +280,9 @@ char   tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
 int32_t tsS3BlockSize = -1;        // number of tsdb pages (4096)
 int32_t tsS3BlockCacheSize = 16;   // number of blocks
 int32_t tsS3PageCacheSize = 4096;  // number of pages
-int32_t tsS3UploadDelaySec = 60 * 60;
+int32_t tsS3UploadDelaySec = 60 * 60 * 24;
+
+bool    tsExperimental = true;
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -355,11 +358,24 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input
   char cfgFile[PATH_MAX + 100] = {0};
 
   taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
+  char  lastC = cfgDir[strlen(cfgDir) - 1];
+  char *tdDirsep = TD_DIRSEP;
+  if (lastC == '\\' || lastC == '/') {
+    tdDirsep = "";
+  }
   if (taosIsDir(cfgDir)) {
 #ifdef CUS_PROMPT
-    snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "%s.cfg", cfgDir, CUS_PROMPT);
+    snprintf(cfgFile, sizeof(cfgFile),
+             "%s"
+             "%s"
+             "%s.cfg",
+             cfgDir, tdDirsep, CUS_PROMPT);
 #else
-    snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
+    snprintf(cfgFile, sizeof(cfgFile),
+             "%s"
+             "%s"
+             "taos.cfg",
+             cfgDir, tdDirsep);
 #endif
   } else {
     tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
@@ -432,6 +448,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   return 0;
 }
 
@@ -508,11 +525,13 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
 
   tsNumOfTaskQueueThreads = tsNumOfCores / 2;
   tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4);
+
   if (tsNumOfTaskQueueThreads >= 50) {
     tsNumOfTaskQueueThreads = 50;
   }
   if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0)
     return -1;
+  if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
 
   return 0;
 }
@@ -692,6 +711,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
       0)
     return -1;
+  if (cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
+      0)
+    return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
       0)
     return -1;
@@ -725,6 +747,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
   if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
     return -1;
+
   if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
@@ -776,6 +799,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
     return -1;
   if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
 
+  if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
+
   GRANT_CFG_ADD;
   return 0;
 }
@@ -954,6 +979,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) {
   tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32;
   metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32;
   stDebugFlag = cfgGetItem(pCfg, "stDebugFlag")->i32;
+  sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32;
 }
 
 static int32_t taosSetSlowLogScope(char *pScope) {
@@ -1060,6 +1086,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
   tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
 
   tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
+
+  tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
   return 0;
 }
 
@@ -1139,6 +1167,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
 
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
+  tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
   tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32;
   tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32;
@@ -1193,6 +1222,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
   tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
 
+  tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
+
   GRANT_CFG_GET;
   return 0;
 }
@@ -1269,7 +1300,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 static int32_t taosCheckGlobalCfg() {
   uint32_t ipv4 = taosGetIpv4FromFqdn(tsLocalFqdn);
   if (ipv4 == 0xffffffff) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+    terrno = TSDB_CODE_RPC_FQDN_ERROR;
     uError("failed to get ip from fqdn:%s since %s, dnode can not be initialized", tsLocalFqdn, terrstr());
     return -1;
   }
@@ -1428,7 +1459,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag},
         {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag},     {"smaDebugFlag", &smaDebugFlag},
         {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag},     {"metaDebugFlag", &metaDebugFlag},
-        {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag},
+        {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag},   {"sndDebugFlag", &sndDebugFlag},
     };
 
     static OptionNameAndVar options[] = {
@@ -1453,6 +1484,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"timeseriesThreshold", &tsTimeSeriesThreshold},
         {"tmqMaxTopicNum", &tmqMaxTopicNum},
         {"transPullupInterval", &tsTransPullupInterval},
+        {"compactPullupInterval", &tsCompactPullupInterval},
         {"trimVDbIntervalSec", &tsTrimVDbIntervalSec},
         {"ttlBatchDropNum", &tsTtlBatchDropNum},
         {"ttlFlushThreshold", &tsTtlFlushThreshold},
@@ -1462,6 +1494,7 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
         {"s3PageCacheSize", &tsS3PageCacheSize},
         {"s3UploadDelaySec", &tsS3UploadDelaySec},
         {"supportVnodes", &tsNumOfSupportVnodes},
+        {"experimental", &tsExperimental}
     };
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@@ -1685,6 +1718,7 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
         {"shellActivityTimer", &tsShellActivityTimer},
         {"slowLogThreshold", &tsSlowLogThreshold},
         {"useAdapter", &tsUseAdapter},
+        {"experimental", &tsExperimental}
     };
 
     if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
@@ -1736,6 +1770,7 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) {
   taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite);
   taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite);
   taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite);
+  taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite);
   uInfo("all debug flag are set to %d", flag);
 }
 

source/common/src/tmsg.c

@@ -3424,6 +3424,66 @@ int32_t tDeserializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq
 
 void tFreeSCompactDbReq(SCompactDbReq *pReq) { FREESQL(); }
 
+int32_t tSerializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+  if (tEncodeI32(&encoder, pRsp->compactId) < 0) return -1;
+  if (tEncodeI8(&encoder, pRsp->bAccepted) < 0) return -1;
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSCompactDbRsp(void *buf, int32_t bufLen, SCompactDbRsp *pRsp) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeI32(&decoder, &pRsp->compactId) < 0) return -1;
+  if (tDecodeI8(&decoder, &pRsp->bAccepted) < 0) return -1;
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  ENCODESQL();
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSKillCompactReq(void *buf, int32_t bufLen, SKillCompactReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  DECODESQL();
+
+  tEndDecode(&decoder);
+
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+void tFreeSKillCompactReq(SKillCompactReq *pReq) { FREESQL(); }
+
 int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) {
   if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1;
   if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1;
@@ -4259,6 +4319,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq
   if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->compactId) < 0) return -1;
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -4276,6 +4337,11 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR
   if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1;
   if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI64(&decoder, &pReq->compactId) < 0) return -1;
+  } else {
+    pReq->compactId = -1;
+  }
 
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
@@ -5051,6 +5117,75 @@ int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) {
   return 0;
 }
 
+int32_t tSerializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSQueryCompactProgressReq(void *buf, int32_t bufLen, SQueryCompactProgressReq *pReq) {
+  int32_t headLen = sizeof(SMsgHead);
+
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, ((uint8_t *)buf) + headLen, bufLen - headLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->numberFileset) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->finished) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSQueryCompactProgressRsp(void *buf, int32_t bufLen, SQueryCompactProgressRsp *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->numberFileset) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->finished) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
 int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) {
   SEncoder encoder = {0};
   tEncoderInit(&encoder, buf, bufLen);
@@ -5139,6 +5274,8 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *
   if (tEncodeI64(&encoder, pReq->tw.skey) < 0) return -1;
   if (tEncodeI64(&encoder, pReq->tw.ekey) < 0) return -1;
 
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -5165,6 +5302,42 @@ int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq
     if (tDecodeI64(&decoder, &pReq->tw.ekey) < 0) return -1;
   }
 
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  }
+
+  tEndDecode(&decoder);
+  tDecoderClear(&decoder);
+  return 0;
+}
+
+int32_t tSerializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
+  SEncoder encoder = {0};
+  tEncoderInit(&encoder, buf, bufLen);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeI32(&encoder, pReq->compactId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1;
+
+  tEndEncode(&encoder);
+
+  int32_t tlen = encoder.pos;
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSVKillCompactReq(void *buf, int32_t bufLen, SVKillCompactReq *pReq) {
+  SDecoder decoder = {0};
+  tDecoderInit(&decoder, buf, bufLen);
+
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  if (tDecodeI32(&decoder, &pReq->compactId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1;
+
   tEndDecode(&decoder);
   tDecoderClear(&decoder);
   return 0;

source/common/src/ttime.c

@@ -174,6 +174,10 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
     i += 2;
   }
 
+  if (hour > 12 || hour < 0) {
+    return -1;
+  }
+
   // return error if there're illegal charaters after min(2 Digits)
   char* minStr = &str[i];
   if (minStr[1] != '\0' && minStr[2] != '\0') {
@@ -181,7 +185,7 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) {
   }
 
   int64_t minute = strnatoi(&str[i], 2);
-  if (minute > 59) {
+  if (minute > 59 || (hour == 12 && minute > 0)) {
     return -1;
   }
 
@@ -646,7 +650,7 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
 
   /* get the basic numeric value */
   int64_t timestamp = taosStr2Int64(token, &endPtr, 10);
-  if (timestamp < 0 || errno != 0) {
+  if ((timestamp == 0 && token[0] != '0') || errno != 0) {
     return -1;
   }
 

source/common/src/tvariant.c

@@ -19,6 +19,278 @@
 #include "ttokendef.h"
 #include "tvariant.h"
 
+int32_t parseBinaryUInteger(const char *z, int32_t n, uint64_t *value) {
+  // skip head 0b
+  const char *p = z + 2;
+  int32_t     l = n - 2;
+
+  while (*p == '0' && l > 0) {
+    p++;
+    l--;
+  }
+  if (l > 64) {  // too big
+    return TSDB_CODE_FAILED;
+  }
+
+  uint64_t val = 0;
+  for (int32_t i = 0; i < l; i++) {
+    val = val << 1;
+    if (p[i] == '1') {
+      val |= 1;
+    } else if (p[i] != '0') {  // abnormal char
+      return TSDB_CODE_FAILED;
+    }
+  }
+  *value = val;
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint64_t *value) {
+  // parse sign
+  bool has_sign = false;
+  if (z[0] == '-') {
+    *is_neg = true;
+    has_sign = true;
+  } else if (z[0] == '+') {
+    has_sign = true;
+  } else if (z[0] != '.' && (z[0] < '0' || z[0] > '9')) {
+    return TSDB_CODE_FAILED;
+  }
+  if (has_sign) {
+    if (n < 2) {
+      return TSDB_CODE_FAILED;
+    }
+    z++;
+    n--;
+  }
+
+  errno = 0;
+  char  *endPtr = NULL;
+  bool parsed = false;
+  if (z[0] == '0' && n > 2) {
+    if (z[1] == 'b' || z[1] == 'B') {
+      // paring as binary
+      return parseBinaryUInteger(z, n, value);
+    }
+
+    if (z[1] == 'x' || z[1] == 'X') {
+      // parsing as hex
+      *value = taosStr2UInt64(z, &endPtr, 16);
+      parsed = true;
+    }
+  }
+
+  if (!parsed) {
+    // parsing as double
+    double val = taosStr2Double(z, &endPtr);
+    if (val > UINT64_MAX) {
+      return TSDB_CODE_FAILED;
+    }
+    *value = round(val);
+  }
+
+  if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+
+int32_t toDoubleEx(const char *z, int32_t n, uint32_t type, double* value) {
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  errno = 0;
+  char* endPtr = NULL;
+  *value = taosStr2Double(z, &endPtr);
+
+  if (errno == ERANGE || errno == EINVAL) {
+    return TSDB_CODE_FAILED;
+  }
+
+  if (endPtr - z != n) {
+    return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  switch (type)
+  {
+    case TK_NK_INTEGER: {
+      *value = taosStr2Int64(z, &endPtr, 10);
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(z, &endPtr));
+      if (!IS_VALID_INT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+  default:
+    break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. try to parse as integer
+  *value = taosStr2Int64(z, &endPtr, 10);
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    // pure decimal part
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr+1 < end && endPtr[1] > '4') {
+        const char *p = z;
+        while (*p == ' ') {
+          p++;
+        }
+        if (*p == '-') {
+          if ( *value > INT64_MIN) {
+            (*value)--;
+          }
+        } else {
+          if ( *value < INT64_MAX) {
+            (*value)++;
+          }
+        }
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool     is_neg = false;
+  uint64_t uv = 0;
+  int32_t  code = parseSignAndUInteger(z, n, &is_neg, &uv);
+  if (code == TSDB_CODE_SUCCESS) {
+    // truncate into int64
+    if (is_neg) {
+      if (uv > 1ull + INT64_MAX) {
+        *value = INT64_MIN;
+        return TSDB_CODE_FAILED;
+      } else {
+        *value = -uv;
+      }
+    } else {
+      if (uv > INT64_MAX) {
+        *value = INT64_MAX;
+        return TSDB_CODE_FAILED;
+      }
+      *value = uv;
+    }
+  }
+
+  return code;
+}
+
+int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
+  errno = 0;
+  char *endPtr = NULL;
+  const char *p = z;
+  while (*p == ' ') {
+    p++;
+  }
+  switch (type) {
+    case TK_NK_INTEGER: {
+      *value = taosStr2UInt64(p, &endPtr, 10);
+      if (*p == '-' && *value) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      return TSDB_CODE_SUCCESS;
+    } break;
+    case TK_NK_FLOAT: {
+      double val = round(taosStr2Double(p, &endPtr));
+      if (!IS_VALID_UINT64(val)) {
+        return TSDB_CODE_FAILED;
+      }
+      if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
+        return TSDB_CODE_FAILED;
+      }
+      *value = val;
+      return TSDB_CODE_SUCCESS;
+    } break;
+    default:
+      break;
+  }
+
+  if (n == 0) {
+    *value = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  // 1. parse as integer
+  *value = taosStr2UInt64(p, &endPtr, 10);
+  if (*p == '-' && *value) {
+    return TSDB_CODE_FAILED;
+  }
+  if (endPtr - z == n) {
+    if (errno == ERANGE || errno == EINVAL) {
+      return TSDB_CODE_FAILED;
+    }
+    return TSDB_CODE_SUCCESS;
+  } else if (errno == 0 && *endPtr == '.') {
+    const char *s = endPtr + 1;
+    const char *end = z + n;
+    bool pure = true;
+    while (s < end) {
+      if (*s < '0' || *s > '9') {
+        pure = false;
+        break;
+      }
+      s++;
+    }
+    if (pure) {
+      if (endPtr + 1 < end && endPtr[1] > '4' && *value < UINT64_MAX) {
+        (*value)++;
+      }
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  // 2. parse as other 
+  bool    is_neg = false;
+  int32_t code = parseSignAndUInteger(z, n, &is_neg, value);
+  if (is_neg) {
+    if (TSDB_CODE_SUCCESS == code && 0 == *value) {
+      return TSDB_CODE_SUCCESS;
+    }
+    return TSDB_CODE_FAILED;
+  }
+  return code;
+}
+
 int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   errno = 0;
   char *endPtr = NULL;
@@ -26,10 +298,10 @@ int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value) {
   *value = taosStr2Int64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
@@ -39,16 +311,15 @@ int32_t toUInteger(const char *z, int32_t n, int32_t base, uint64_t *value) {
   const char *p = z;
   while (*p == ' ') p++;
   if (*p == '-') {
-    return -1;
+    return TSDB_CODE_FAILED;
   }
 
   *value = taosStr2UInt64(z, &endPtr, base);
   if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
     errno = 0;
-    return -1;
+    return TSDB_CODE_FAILED;
   }
-
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
 /**
@@ -147,7 +418,6 @@ void taosVariantDestroy(SVariant *pVar) {
     taosMemoryFreeClear(pVar->pz);
     pVar->nLen = 0;
   }
-
 }
 
 void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
@@ -172,7 +442,6 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
   if (IS_NUMERIC_TYPE(pSrc->nType) || (pSrc->nType == TSDB_DATA_TYPE_BOOL)) {
     pDst->i = pSrc->i;
   }
-
 }
 
 int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2) {