Merge branch '3.0' of https://github.com/taosdata/TDengine into test/3.0/TD-29957

cmake/cmake.options

@@ -47,27 +47,20 @@ IF(${TD_WINDOWS})
 
     MESSAGE("build wingetopt Win32")
     option(
-            BUILD_WINGETOPT 
+            BUILD_WINGETOPT
                 "If build wingetopt on Windows"
             ON
         )
-    
+
     option(
             TDENGINE_3
             "TDengine 3.x for taos-tools"
             ON
         )
-    
+
     option(
             BUILD_CRASHDUMP
-            "If build crashdump on Windows" 
+            "If build crashdump on Windows"
-            ON
-        )
-
-    MESSAGE("build geos Win32")
-    option(
-            BUILD_GEOS
-            "If build geos on Windows"
             ON
         )
 
@@ -79,7 +72,7 @@ ENDIF ()
 
 option(
         BUILD_GEOS
-        "If build geos on Windows"
+        "If build with geos"
         ON
     )
 
@@ -95,6 +88,12 @@ option(
     ON
     )
 
+option(
+    BUILD_PCRE2
+    "If build with pcre2"
+    ON
+)
+
 option(
     JEMALLOC_ENABLED
     "If build with jemalloc"
@@ -114,14 +113,14 @@ option(
     )
 
 option(
-    BUILD_WITH_LEVELDB 
+    BUILD_WITH_LEVELDB
-    "If build with leveldb" 
+    "If build with leveldb"
     OFF
 )
 
 option(
-    BUILD_WITH_ROCKSDB 
+    BUILD_WITH_ROCKSDB
-    "If build with rocksdb" 
+    "If build with rocksdb"
     ON
 )
 
@@ -170,46 +169,46 @@ ENDIF ()
 
 option(
     BUILD_WITH_SQLITE
-    "If build with sqlite" 
+    "If build with sqlite"
     OFF
 )
 
 option(
     BUILD_WITH_BDB
-    "If build with BDB" 
+    "If build with BDB"
     OFF
 )
 
 option(
-    BUILD_WITH_LUCENE 
+    BUILD_WITH_LUCENE
-    "If build with lucene" 
+    "If build with lucene"
-    off 
+    off
 )
 
 
 option(
     BUILD_WITH_NURAFT
-    "If build with NuRaft" 
+    "If build with NuRaft"
     OFF
 )
 
 option(
     BUILD_WITH_UV
-    "If build with libuv" 
+    "If build with libuv"
-    ON  
+    ON
 )
 
 option(
     BUILD_WITH_UV_TRANS
-    "If build with libuv_trans " 
+    "If build with libuv_trans "
-    ON 
+    ON
 )
 
 IF(${TD_LINUX} MATCHES TRUE)
 
 option(
-    BUILD_DEPENDENCY_TESTS 
+    BUILD_DEPENDENCY_TESTS
-    "If build dependency tests" 
+    "If build dependency tests"
     ON
 )
 
@@ -217,14 +216,14 @@ ENDIF ()
 
 option(
     BUILD_DOCS
-    "If use doxygen build documents" 
+    "If use doxygen build documents"
     OFF
 )
 
 option(
    BUILD_WITH_INVERTEDINDEX
-   "If use invertedIndex" 
+   "If use invertedIndex"
-   ON 
+   ON
 )
 
 option(

cmake/curl_CMakeLists.txt.in

@@ -12,7 +12,7 @@ ExternalProject_Add(curl2
         BUILD_IN_SOURCE TRUE
         BUILD_ALWAYS 1
         UPDATE_COMMAND ""
-        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 #--enable-debug
+        CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2  --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 --without-libpsl #--enable-debug
         BUILD_COMMAND make -j
         INSTALL_COMMAND make install
         TEST_COMMAND ""

cmake/pcre2_CMakeLists.txt.in

@@ -0,0 +1,13 @@
+
+# pcre2
+ExternalProject_Add(pcre2
+        GIT_REPOSITORY https://github.com/PCRE2Project/pcre2.git
+        GIT_TAG pcre2-10.43
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pcre2"
+        #BINARY_DIR "${TD_CONTRIB_DIR}/pcre2"
+        #BUILD_IN_SOURCE TRUE
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND ""
+        INSTALL_COMMAND ""
+        TEST_COMMAND ""
+        )

cmake/xz_CMakeLists.txt.in

@@ -1,7 +1,5 @@
 
-# xz 
+# xz
-
-if (${TD_LINUX})
 ExternalProject_Add(xz
         GIT_REPOSITORY https://github.com/xz-mirror/xz.git 
         GIT_TAG v5.4.4 
@@ -13,5 +11,4 @@ ExternalProject_Add(xz
         BUILD_COMMAND ""
         INSTALL_COMMAND ""
         TEST_COMMAND ""
-)
+)
-endif()

cmake/zlib_CMakeLists.txt.in

@@ -2,7 +2,6 @@
 # zlib
 
 
-if (${TD_LINUX})
 ExternalProject_Add(zlib
         GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
         GIT_TAG v1.2.11
@@ -14,4 +13,3 @@ ExternalProject_Add(zlib
         INSTALL_COMMAND ""
         TEST_COMMAND ""
 )
-endif()

contrib/CMakeLists.txt

@@ -121,7 +121,7 @@ if (${BUILD_CONTRIB})
         cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
         add_definitions(-DUSE_ROCKSDB)
     endif()
-else() 
+else()
     if (NOT ${TD_LINUX})
         if(${BUILD_WITH_ROCKSDB})
             cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
@@ -183,6 +183,11 @@ if(${BUILD_GEOS})
     cat("${TD_SUPPORT_DIR}/geos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
+#
+if(${BUILD_PCRE2})
+    cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+endif()
+
 # download dependencies
 configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -250,6 +255,10 @@ target_include_directories(
 
 # zlib
 set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
+if(${TD_DARWIN})
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=deprecated-non-prototype")
+    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=deprecated-non-prototype")
+endif(${TD_DARWIN})
 add_subdirectory(zlib EXCLUDE_FROM_ALL)
 target_include_directories(
     zlibstatic
@@ -266,8 +275,8 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
 # add_subdirectory(xz EXCLUDE_FROM_ALL)
 # target_include_directories(
 #     xz
-#     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz 
+#     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz
-#     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz 
+#     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz
 # )
 
 # leveldb
@@ -347,7 +356,7 @@ if (${BUILD_WITH_ROCKSDB})
             rocksdb
             PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
         )
-    else() 
+    else()
         if (NOT ${TD_LINUX})
             MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS})
             MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS})
@@ -396,8 +405,8 @@ if (${BUILD_WITH_ROCKSDB})
                 rocksdb
                 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
             )
-        endif() 
+        endif()
-      
+
     endif()
 endif()
 
@@ -601,6 +610,10 @@ if(${BUILD_GEOS})
     )
 endif(${BUILD_GEOS})
 
+if (${BUILD_PCRE2})
+    add_subdirectory(pcre2 EXCLUDE_FROM_ALL)
+endif(${BUILD_PCRE2})
+
 # ================================================================================================
 # Build test
 # ================================================================================================

docs/en/01-index.md

@@ -13,7 +13,7 @@ TDengine greatly improves the efficiency of data ingestion, querying, and storag
 
 If you are a developer, please read the [Developer Guide](./develop) carefully. This section introduces the database connection, data modeling, data ingestion, query, continuous query, cache, data subscription, user-defined functions, and other functionality in detail. Sample code is provided for a variety of programming languages. In most cases, you can just copy and paste the sample code, and make a few changes to accommodate your application, and it will work.
 
-We live in the era of big data, and scale-up is unable to meet the growing needs of the business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to [Cluster Deployment](../deployment).
+We live in the era of big data, and scale-up is unable to meet the growing needs of the business. Any modern data system must have the ability to scale out, and clustering has become an indispensable feature of big data systems. Not only did the TDengine team develop the cluster feature, but also decided to open source this important feature. To learn how to deploy, manage and maintain a TDengine cluster please refer to [Cluster Deployment](./deployment).
 
 TDengine uses ubiquitous SQL as its query language, which greatly reduces learning costs and migration costs. In addition to the standard SQL, TDengine has extensions to better support time series data analysis. These extensions include functions such as roll-up, interpolation, and time-weighted average, among many others. The [SQL Reference](./taos-sql) chapter describes the SQL syntax in detail and lists the various supported commands and functions.
 

docs/en/07-develop/01-connect/index.md

@@ -36,7 +36,7 @@ For REST and native connections, client libraries provide similar APIs for perfo
 Key differences:
 
 3. The REST connection is more accessible with cross-platform support, however it results in a 30% performance downgrade.
-1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](../../client-libraries/cpp#parameter-binding-api), [Subscription](../../client-libraries/cpp#subscription-and-consumption-api), etc.
+1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](../../client-libraries/cpp#parameter-binding-api), [Subscription](../../client-libraries/cpp#subscription-api), etc.
 
 ## Install Client Driver taosc
 

docs/en/07-develop/02-model/index.mdx

@@ -41,7 +41,7 @@ Without the current database specified, table name must be preceded with the cor
 
 ## Create STable
 
-In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](../../concept/#model_table1), the SQL statement below can be used to create the super table.
+In a time-series application, there may be multiple kinds of data collection points. For example, in the electrical power system there are meters, transformers, bus bars, switches, etc. For easy and efficient aggregation of multiple tables, one STable needs to be created for each kind of data collection point. For example, for the meters in [table 1](../../concept/), the SQL statement below can be used to create the super table.
 
 ```sql
 CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int);

docs/en/07-develop/03-insert-data/30-influxdb-line.mdx

@@ -46,7 +46,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0
 
 :::
 
-For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](../../../reference/schemaless/#Schemaless-Line-Protocol)
+For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](../../../reference/schemaless/)
 
 ## Examples
 

docs/en/08-client-libraries/03-cpp.mdx

@@ -32,7 +32,7 @@ The version number of the TDengine client driver and the version number of the T
 
 ## Installation Steps
 
-Please refer to the [Installation Steps](../#installation-steps) for TDengine client driver installation
+Please refer to [Install Client Driver](../#install-client-driver) for TDengine client driver installation
 
 ## Establishing a connection
 
@@ -372,7 +372,7 @@ The specific functions related to the interface are as follows (see also the [pr
   Execute the prepared statement. Currently, a statement can only be executed once.
 
 - `int taos_stmt_affected_rows(TAOS_STMT *stmt)`
- 
+
   Gets the number of rows affected by executing bind statements multiple times.
 
 - `int taos_stmt_affected_rows_once(TAOS_STMT *stmt)`

docs/en/08-client-libraries/04-java.mdx

@@ -160,7 +160,7 @@ For Java connector, the jts library can be used to easily create GEOMETRY type o
 Before using Java client library to connect to the database, the following conditions are required.
 
 - Java 1.8 or above runtime environment and Maven 3.6 or above installed
-- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Installing Client Driver](../#Install-Client-Driver)
+- TDengine client driver installed (required for native connections, not required for REST connections), please refer to [Install Client Driver](../#install-client-driver)
 
 ### Install the client library
 
@@ -368,7 +368,7 @@ The configuration parameters in properties are as follows.
 - TSDBDriver.PROPERTY_KEY_MESSAGE_WAIT_TIMEOUT: message transmission timeout in milliseconds, the default value is 60000 ms. It only takes effect when using JDBC REST connection and batchfetch is true.
 - TSDBDriver.PROPERTY_KEY_USE_SSL: connecting Securely Using SSL. true: using SSL connection, false: not using SSL connection. It only takes effect when using JDBC REST connection.
 - TSDBDriver.HTTP_POOL_SIZE: size of REST concurrent requests. The default value is 20.
-  For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/#Client-Only).
+  For JDBC native connections, you can specify other parameters, such as log level, SQL length, etc., by specifying URL and Properties. For more detailed configuration, please refer to [Client Configuration](../../reference/config/#configuration-file-on-client-side).
 
 ### Priority of configuration parameters
 

docs/en/08-client-libraries/05-go.mdx

@@ -129,7 +129,7 @@ username:password@protocol(address)/dbname?param=value
 
 _taosSql_ implements Go's `database/sql/driver` interface via cgo. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver.
 
-Use `taosSql` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName`, DSN supports the following parameters.
+Use `taosSql` as `driverName` and use a correct DSN as `dataSourceName`, DSN supports the following parameters.
 
 * cfg specifies the `taos.cfg` directory
 
@@ -160,7 +160,7 @@ func main() {
 
 _taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver.
 
-Use `taosRestful` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN.
+Use `taosRestful` as `driverName` and use a correct DSN as `dataSourceName` with the following parameters supported by the DSN.
 
 * `disableCompression` whether to accept compressed data, default is true do not accept compressed data, set to false if transferring data using gzip compression.
 * `readBufferSize` The default size of the buffer for reading data is 4K (4096), which can be adjusted upwards when the query result has a lot of data.
@@ -191,7 +191,7 @@ func main() {
 
 _taosRestful_ implements Go's `database/sql/driver` interface via `http client`. You can use the [`database/sql`](https://golang.org/pkg/database/sql/) interface by simply introducing the driver (driver-go minimum version 3.0.2).
 
-Use `taosWS` as `driverName` and use a correct [DSN](#DSN) as `dataSourceName` with the following parameters supported by the DSN.
+Use `taosWS` as `driverName` and use a correct DSN as `dataSourceName` with the following parameters supported by the DSN.
 
 * `writeTimeout` The timeout to send data via WebSocket.
 * `readTimeout` The timeout to receive response data via WebSocket.

docs/en/08-client-libraries/06-rust.mdx

@@ -321,7 +321,7 @@ let rs = taos.query_with_req_id("select * from stable where tag1 is null", 1)?;
 
 TDengine has significantly improved the bind APIs to support data writing (INSERT) scenarios. Writing data in this way avoids the resource consumption of SQL syntax parsing, resulting in significant write performance improvements in many cases.
 
-Parameter binding details see [API Reference](#stmt-api)
+Parameter binding details see [API Reference](#bind-interface)
 
 <RustBind />
 
@@ -447,7 +447,7 @@ The following parameters can be configured for the TMQ DSN. Only `group.id` is m
 - `auto.commit.interval.ms`: Interval for automatic commits.
 
 #### Full Sample Code
-  
+
 For more information, see [GitHub sample file](https://github.com/taosdata/TDengine/blob/3.0/docs/examples/rust/nativeexample/examples/subscribe_demo.rs).
 
 ### Use with connection pool
@@ -559,7 +559,7 @@ The [Taos][struct.Taos] object provides an API to perform operations on multiple
         // binary/varchar to String
         location: String,
     }
-
+    
     let records: Vec<Record> = taos
         .query("select * from `meters`")
         .await?
@@ -578,11 +578,7 @@ Note that Rust asynchronous functions and an asynchronous runtime are required.
 
 In addition, this structure is also the entry point for Parameter Binding and Line Protocol Interface. Please refer to the specific API descriptions for usage.
 
-<p>
+### Bind Interface
-<a id="stmt-api" style={{color:'#141414'}}>
-Bind Interface
-</a>
-</p>
 
 Similar to the C interface, Rust provides the bind interface's wrapping. First, the [Taos][struct.taos] object creates a parameter binding object [Stmt] for an SQL statement.
 

docs/en/08-client-libraries/07-python.mdx

@@ -95,7 +95,7 @@ TDengine currently supports timestamp, number, character, Boolean type, and the
 
 1. Install Python. The recent taospy package requires Python 3.6.2+. The earlier versions of taospy require Python 3.7+.  The taos-ws-py package requires Python 3.7+. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it.
 2. Install [pip](https://pypi.org/project/pip/). In most cases, the Python installer comes with the pip utility. If not, please refer to [pip documentation](https://pip.pypa.io/en/stable/installation/) to install it.
-If you use a native connection, you will also need to [Install Client Driver](../#Install-Client-Driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI.
+If you use a native connection, you will also need to [Install Client Driver](../#install-client-driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI.
 
 ### Install via pip
 

docs/en/08-client-libraries/08-node.mdx

@@ -58,7 +58,7 @@ Please refer to [version support list](../#version-support)
 ### Pre-installation preparation
 
 - Install the Node.js development environment
-- If you are using the REST client library, skip this step. However, if you use the native client library, please install the TDengine client driver. Please refer to [Install Client Driver](../#Install-Client-Driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS.
+- If you are using the REST client library, skip this step. However, if you use the native client library, please install the TDengine client driver. Please refer to [Install Client Driver](../#install-client-driver) for more details. We use [node-gyp](https://github.com/nodejs/node-gyp) to interact with TDengine instances and also need to install some dependencies mentioned below depending on the specific OS.
 
 <Tabs defaultValue="Linux">
 <TabItem value="Linux" label="Linux system installation dependencies">

docs/en/08-client-libraries/80-php.mdx

@@ -40,7 +40,7 @@ Because the version of TDengine client driver is tightly associated with that of
 
 ### Install TDengine Client Driver
 
-Regarding how to install TDengine client driver please refer to [Install Client Driver](../#installation-steps)
+Regarding how to install TDengine client driver please refer to [Install Client Driver](../#install-client-driver)
 
 ### Install php-tdengine
 

docs/en/14-reference/05-taosbenchmark.md

@@ -92,7 +92,7 @@ taosBenchmark -f <json file>
 
 </details>
 
-## Command-line argument in detail
+## Command-line arguments in detail
 
 - **-f/--file &lt;json file&gt;** :
   specify the configuration file to use. This file includes All parameters. Users should not use this parameter with other parameters on the command-line. There is no default value.

docs/en/14-reference/14-taosKeeper.md

@@ -23,7 +23,7 @@ Methods of installing taosKeeper:
 
 ### Configuration
 
-taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: [Command-line arguments](#command-line-arguments-in-detail), [environment variable](#environment-variable-in-detail) and [configuration file](#configuration-file-parameters-in-detail). The precedence of those is Command-line, environment variable and configuration file.
+taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: Command-line arguments, environment variable and configuration file. The precedence of those is Command-line, environment variable and configuration file.
 
 **Make sure that the TDengine cluster is running correctly before running taosKeeper.** Ensure that the monitoring service in TDengine has been started. At least the values of `monitor` and `monitorFqdn` need to be set in `taos.cfg`.
 
@@ -32,7 +32,7 @@ monitor 1
 monitorFqdn localhost # taoskeeper's FQDN
 ```
 
-For more information, see [TDengine Monitoring Configuration](../config/#monitoring).
+For more information, see [TDengine Monitoring Configuration](../config/#monitoring-parameters).
 
 ### Quick Launch
 

docs/en/20-third-party/03-telegraf.md

@@ -73,6 +73,6 @@ Query OK, 3 row(s) in set (0.013269s)
 
 - TDengine take influxdb format data and create unique ID for table names by the rule.
 The user can configure `smlChildTableName` parameter to generate specified table names if he/she needs. And he/she also need to insert data with specified data format.
-For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](../../reference/schemaless/#Schemaless-Line-Protocol)
+For example, Add `smlChildTableName=tname` in the taos.cfg file. Insert data `st,tname=cpu1,t1=4 c1=3 1626006833639000000` then the table name will be cpu1. If there are multiple lines has same tname but different tag_set, the first line's tag_set will be used to automatically creating table and ignore other lines. Please refer to [TDengine Schemaless](../../reference/schemaless/)
 :::
 

include/common/tdataformat.h

@@ -95,6 +95,15 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
 #define COL_VAL_IS_NULL(CV)  ((CV)->flag == CV_FLAG_NULL)
 #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE)
 
+#define tRowGetKey(_pRow, _pKey)           \
+  do {                                     \
+    (_pKey)->ts = (_pRow)->ts;             \
+    (_pKey)->numOfPKs = 0;                 \
+    if ((_pRow)->numOfPKs > 0) {           \
+      tRowGetPrimaryKey((_pRow), (_pKey)); \
+    }                                      \
+  } while (0)
+
 // SValueColumn ================================
 typedef struct {
   int8_t  cmprAlg;  // filled by caller
@@ -125,8 +134,8 @@ void    tRowDestroy(SRow *pRow);
 int32_t tRowSort(SArray *aRowP);
 int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
 int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
-void    tRowGetKey(SRow *pRow, SRowKey *key);
+void    tRowGetPrimaryKey(SRow *pRow, SRowKey *key);
-int32_t tRowKeyCompare(const void *p1, const void *p2);
+int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2);
 int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
 
 // SRowIter ================================

include/common/tglobal.h

@@ -258,7 +258,7 @@ extern bool tsExperimental;
 int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd,
                       const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
 int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
-                    bool tsc);
+                    bool tsc, bool isDumpCfg);
 void    taosCleanupCfg();
 
 int32_t taosCfgDynamicOptions(SConfig *pCfg, const char *name, bool forServer);

include/common/tgrant.h

@@ -56,6 +56,10 @@ typedef enum {
   TSDB_GRANT_VIEW,
   TSDB_GRANT_MULTI_TIER,
   TSDB_GRANT_BACKUP_RESTORE,
+  TSDB_GRANT_OBJECT_STORAGE,
+  TSDB_GRANT_ACTIVE_ACTIVE,
+  TSDB_GRANT_DUAL_REPLICA_HA,
+  TSDB_GRANT_DB_ENCRYPTION,
 } EGrantType;
 
 int32_t checkAndGetCryptKey(const char *encryptCode, const char *machineId, char **key);

include/common/tmsg.h

@@ -329,6 +329,7 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_DB_ALIVE_STMT,
   QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT,
   QUERY_NODE_BALANCE_VGROUP_LEADER_STMT,
+  QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT,
   QUERY_NODE_RESTORE_DNODE_STMT,
   QUERY_NODE_RESTORE_QNODE_STMT,
   QUERY_NODE_RESTORE_MNODE_STMT,
@@ -2425,10 +2426,11 @@ int32_t tDeserializeSRedistributeVgroupReq(void* buf, int32_t bufLen, SRedistrib
 void    tFreeSRedistributeVgroupReq(SRedistributeVgroupReq* pReq);
 
 typedef struct {
-  int32_t useless;
+  int32_t reserved;
   int32_t vgId;
   int32_t sqlLen;
   char*   sql;
+  char    db[TSDB_DB_FNAME_LEN];
 } SBalanceVgroupLeaderReq;
 
 int32_t tSerializeSBalanceVgroupLeaderReq(void* buf, int32_t bufLen, SBalanceVgroupLeaderReq* pReq);

include/libs/executor/storageapi.h

@@ -270,7 +270,7 @@ typedef struct SStoreMeta {
   int32_t (*putCachedTableList)(void* pVnode, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload,
                                 int32_t payloadLen, double selectivityRatio);
 
-  void* (*storeGetIndexInfo)();
+  void* (*storeGetIndexInfo)(void *pVnode);
   void* (*getInvertIndex)(void* pVnode);
   // support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
   int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);

include/libs/nodes/cmdnodes.h

@@ -582,6 +582,7 @@ typedef struct SBalanceVgroupStmt {
 typedef struct SBalanceVgroupLeaderStmt {
   ENodeType type;
   int32_t   vgId;
+  char      dbName[TSDB_DB_NAME_LEN];
 } SBalanceVgroupLeaderStmt;
 
 typedef struct SMergeVgroupStmt {

include/os/osSemaphore.h

@@ -33,6 +33,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
 int tsem_post(tsem_t *sem);
 int tsem_destroy(tsem_t *sem);
 
+#define tsem2_t        tsem_t
+#define tsem2_init     tsem_init
+#define tsem2_wait     tsem_wait
+#define tsem2_timewait tsem_timewait
+#define tsem2_post     tsem_post
+#define tsem2_destroy  tsem_destroy
+
 #elif defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
 #include <windows.h>
 
@@ -44,6 +51,13 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
 int tsem_post(tsem_t *sem);
 int tsem_destroy(tsem_t *sem);
 
+#define tsem2_t        tsem_t
+#define tsem2_init     tsem_init
+#define tsem2_wait     tsem_wait
+#define tsem2_timewait tsem_timewait
+#define tsem2_post     tsem_post
+#define tsem2_destroy  tsem_destroy
+
 #else
 
 #define tsem_t       sem_t
@@ -53,6 +67,20 @@ int tsem_timewait(tsem_t *sim, int64_t milis);
 #define tsem_post    sem_post
 #define tsem_destroy sem_destroy
 
+typedef struct tsem2_t {
+  TdThreadMutex    mutex;
+  TdThreadCond     cond;
+  TdThreadCondAttr attr;
+  int              count;
+} tsem2_t;
+
+// #define tsem2_t       sem_t
+int tsem2_init(tsem2_t* sem, int pshared, unsigned int value);
+int tsem2_wait(tsem2_t* sem);
+int tsem2_timewait(tsem2_t* sem, int64_t milis);
+int tsem2_post(tsem2_t* sem);
+int tsem2_destroy(tsem2_t* sem);
+
 #endif
 
 #if defined(_TD_DARWIN_64)

include/os/osString.h

@@ -22,7 +22,11 @@ extern "C" {
 
 typedef wchar_t TdWchar;
 typedef int32_t TdUcs4;
+#if !defined(DISALLOW_NCHAR_WITHOUT_ICONV) && defined(DARWIN)
+#include "iconv.h"
+#else
 typedef void   *iconv_t;
+#endif
 typedef enum { M2C = 0, C2M } ConvType;
 
 // If the error is in a third-party library, place this header file under the third-party library header file.

include/os/osThread.h

@@ -222,6 +222,7 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const
 int32_t taosThreadCondAttrDestroy(TdThreadCondAttr *attr);
 int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr *attr, int32_t *pshared);
 int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr);
+int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId);
 int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared);
 int32_t taosThreadDetach(TdThread thread);
 int32_t taosThreadEqual(TdThread t1, TdThread t2);

include/util/taoserror.h

@@ -327,6 +327,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_DB_IN_CREATING            TAOS_DEF_ERROR_CODE(0, 0x0396) //
 #define TSDB_CODE_MND_INVALID_SYS_TABLENAME     TAOS_DEF_ERROR_CODE(0, 0x039A)
 #define TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE  TAOS_DEF_ERROR_CODE(0, 0x039B)
+#define TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED  TAOS_DEF_ERROR_CODE(0, 0x039C)
 
 // mnode-node
 #define TSDB_CODE_MND_MNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x03A0)

include/util/tgeosctx.h

@@ -21,6 +21,7 @@ extern "C" {
 #endif
 
 #include <geos_c.h>
+#include <tpcre2.h>
 
 typedef struct SGeosContext {
   GEOSContextHandle_t handle;
@@ -31,6 +32,9 @@ typedef struct SGeosContext {
   GEOSWKBReader *WKBReader;
   GEOSWKBWriter *WKBWriter;
 
+  pcre2_code *WKTRegex;
+  pcre2_match_data *WKTMatchData;
+
   char errMsg[512];
 } SGeosContext;
 

include/util/tpcre2.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_ULIT_PCRE2_H_
+#define _TD_ULIT_PCRE2_H_
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include "pcre2.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t doRegComp(pcre2_code** ppRegex, pcre2_match_data** ppMatchData, const char* pattern);
+int32_t doRegExec(const char* pString, pcre2_code* pRegex, pcre2_match_data* pMatchData);
+void    destroyRegexes(pcre2_code* pWktRegex, pcre2_match_data* pWktMatchData);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // _TD_UTIL_PAGEDBUF_H_

packaging/tools/makepkg.sh

@@ -101,6 +101,7 @@ else
       ${script_dir}/remove.sh \
       ${script_dir}/set_core.sh \
       ${script_dir}/startPre.sh \
+      ${script_dir}/quick_deploy.sh \
       ${script_dir}/taosd-dump-cfg.gdb"
 fi
 
@@ -258,7 +259,7 @@ cp ${install_files} ${install_dir}
 cp ${install_dir}/install.sh install_temp.sh
 if [ "$verMode" == "cluster" ]; then
   sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh
-  sed -i "s/PREFIX=\"taos\"/PREFIX=\"${serverName2}\"/g" install_temp.sh    
+  sed -i "s/PREFIX=\"taos\"/PREFIX=\"${clientName2}\"/g" install_temp.sh
   sed -i "s/productName=\"TDengine\"/productName=\"${productName2}\"/g" install_temp.sh
   cusDomain=`echo "${cusEmail2}" | sed 's/^[^@]*@//'`
   sed -i "s/emailName=\"taosdata.com\"/emailName=\"${cusDomain}\"/g" install_temp.sh

source/client/CMakeLists.txt

@@ -23,13 +23,6 @@ target_link_libraries(
     PRIVATE os util common transport monitor nodes parser command planner catalog scheduler function qcom geometry
 )
 
-if(TD_DARWIN_ARM64)
-    target_link_libraries(
-        taos
-        PRIVATE  "-arch x86_64"
-    )
-endif()
-
 if(TD_WINDOWS)
     INCLUDE_DIRECTORIES(jni/windows)
     INCLUDE_DIRECTORIES(jni/windows/win32)

source/client/src/clientEnv.c

@@ -725,7 +725,7 @@ void taos_init_imp(void) {
     return;
   }
 
-  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1, true) != 0) {
     tscInitRes = -1;
     return;
   }

source/client/src/clientHb.c

@@ -738,7 +738,7 @@ static int32_t hbGetUserAuthInfo(SClientHbKey *connKey, SHbParam *param, SClient
 _return:
   releaseTscObj(connKey->tscRid);
   if (code) {
-    tscError("hb got user auth info failed since %s", terrstr(code));
+    tscError("hb got user auth info failed since %s", tstrerror(code));
   }
 
   return code;

source/client/src/clientRawBlockWrite.c

@@ -30,6 +30,11 @@ static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3
 
 static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id,
                                   int8_t t, SColCmprWrapper* pColCmprRow) {
+  int8_t buildDefaultCompress = 0;
+  if (pColCmprRow->nCols <= 0) {
+    buildDefaultCompress = 1;
+  }
+
   char*  string = NULL;
   cJSON* json = cJSON_CreateObject();
   if (json == NULL) {
@@ -70,13 +75,20 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch
     cJSON_AddItemToObject(column, "isPrimarykey", isPk);
     cJSON_AddItemToArray(columns, column);
 
-    if (pColCmprRow == NULL || pColCmprRow->nCols <= i) {
+    if (pColCmprRow == NULL) {
       continue;
     }
-    SColCmpr*   pColCmpr = pColCmprRow->pColCmpr + i;
+
-    const char* encode = columnEncodeStr(COMPRESS_L1_TYPE_U32(pColCmpr->alg));
+    uint32_t alg = 0;
-    const char* compress = columnCompressStr(COMPRESS_L2_TYPE_U32(pColCmpr->alg));
+    if (buildDefaultCompress) {
-    const char* level = columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(pColCmpr->alg));
+      alg = createDefaultColCmprByType(s->type);
+    } else {
+      SColCmpr* pColCmpr = pColCmprRow->pColCmpr + i;
+      alg = pColCmpr->alg;
+    }
+    const char* encode = columnEncodeStr(COMPRESS_L1_TYPE_U32(alg));
+    const char* compress = columnCompressStr(COMPRESS_L2_TYPE_U32(alg));
+    const char* level = columnLevelStr(COMPRESS_L2_TYPE_LEVEL_U32(alg));
 
     cJSON* encodeJson = cJSON_CreateString(encode);
     cJSON_AddItemToObject(column, "encode", encodeJson);
@@ -767,14 +779,25 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
     code = TSDB_CODE_INVALID_PARA;
     goto end;
   }
+
+  int8_t           createDefaultCompress = 0;
+  SColCmprWrapper* p = &req.colCmpr;
+  if (p->nCols == 0) {
+    createDefaultCompress = 1;
+  }
   // build create stable
   pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
   for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
     SSchema*          pSchema = req.schemaRow.pSchema + i;
     SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
     strcpy(field.name, pSchema->name);
-    SColCmpr *p = &req.colCmpr.pColCmpr[i];
+
-    field.compress = p->alg;
+    if (createDefaultCompress) {
+      field.compress = createDefaultColCmprByType(pSchema->type);
+    } else {
+      SColCmpr* p = &req.colCmpr.pColCmpr[i];
+      field.compress = p->alg;
+    }
     taosArrayPush(pReq.pColumns, &field);
   }
   pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));

source/client/src/clientTmq.c

@@ -105,7 +105,7 @@ struct tmq_t {
   STaosQueue* mqueue;        // queue of rsp
   STaosQall*  qall;
   STaosQueue* delayedTask;  // delayed task queue for heartbeat and auto commit
-  tsem_t      rspSem;
+  tsem2_t      rspSem;
 };
 
 typedef struct SAskEpInfo {
@@ -727,7 +727,7 @@ static void generateTimedTask(int64_t refId, int32_t type) {
 
   *pTaskType = type;
   taosWriteQitem(tmq->delayedTask, pTaskType);
-  tsem_post(&tmq->rspSem);
+  tsem2_post(&tmq->rspSem);
   taosReleaseRef(tmqMgmt.rsetId, refId);
 }
 
@@ -742,7 +742,7 @@ void tmqReplayTask(void* param, void* tmrId) {
   tmq_t*  tmq = taosAcquireRef(tmqMgmt.rsetId, refId);
   if (tmq == NULL) goto END;
 
-  tsem_post(&tmq->rspSem);
+  tsem2_post(&tmq->rspSem);
   taosReleaseRef(tmqMgmt.rsetId, refId);
 END:
   taosMemoryFree(param);
@@ -1033,7 +1033,7 @@ void tmqFreeImpl(void* handle) {
   }
 
   taosFreeQall(tmq->qall);
-  tsem_destroy(&tmq->rspSem);
+  tsem2_destroy(&tmq->rspSem);
 
   taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl);
   taos_close_internal(tmq->pTscObj);
@@ -1121,7 +1121,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->consumerId = tGenIdPI64();
 
   // init semaphore
-  if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
+  if (tsem2_init(&pTmq->rspSem, 0, 0) != 0) {
     tscError("consumer:0x %" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(),
              pTmq->groupId);
     SET_ERROR_MSG_TMQ("init t_sem failed")
@@ -1132,7 +1132,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
   if (pTmq->pTscObj == NULL) {
     tscError("consumer:0x%" PRIx64 " setup failed since %s, groupId:%s", pTmq->consumerId, terrstr(), pTmq->groupId);
-    tsem_destroy(&pTmq->rspSem);
+    tsem2_destroy(&pTmq->rspSem);
     SET_ERROR_MSG_TMQ("init tscObj failed")
     goto _failed;
   }
@@ -1434,7 +1434,7 @@ END:
   taosReleaseRef(tmqMgmt.rsetId, refId);
 
 FAIL:
-  if (tmq) tsem_post(&tmq->rspSem);
+  if (tmq) tsem2_post(&tmq->rspSem);
   taosMemoryFree(pParam);
   if (pMsg) taosMemoryFreeClear(pMsg->pData);
   if (pMsg) taosMemoryFreeClear(pMsg->pEpSet);
@@ -2132,10 +2132,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) {
                  tmq->consumerId, tmq->epoch, startTime, currentTime);
         return NULL;
       }
-      tsem_timewait(&tmq->rspSem, (timeout - elapsedTime));
+      tsem2_timewait(&tmq->rspSem, (timeout - elapsedTime));
     } else {
       // use tsem_timewait instead of tsem_wait to avoid unexpected stuck
-      tsem_timewait(&tmq->rspSem, 1000);
+      tsem2_timewait(&tmq->rspSem, 1000);
     }
   }
 }

source/common/src/cos.c

@@ -27,6 +27,8 @@ static S3UriStyle uriStyleG = S3UriStylePath;
 static int        retriesG = 5;
 static int        timeoutMsG = 0;
 
+extern int8_t tsS3Oss;
+
 int32_t s3Begin() {
   S3Status    status;
   const char *hostname = tsS3Hostname;
@@ -42,6 +44,9 @@ int32_t s3Begin() {
   }
 
   protocolG = !tsS3Https;
+  if (tsS3Oss) {
+    uriStyleG = S3UriStyleVirtualHost;
+  }
 
   return 0;
 }
@@ -58,6 +63,11 @@ static int32_t s3ListBucket(char const *bucketname);
 int32_t s3CheckCfg() {
   int32_t code = 0;
 
+  if (!tsS3Enabled) {
+    fprintf(stderr, "s3 not configured.\n");
+    goto _exit;
+  }
+
   code = s3Begin();
   if (code != 0) {
     fprintf(stderr, "failed to initialize s3.\n");

source/common/src/tcol.c

@@ -250,7 +250,7 @@ bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OP
     strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
     return true;
   }
-  return checkColumnEncode(encode);
+  return checkColumnEncode(encode) && validColEncode(type, columnEncodeVal(encode));
 }
 bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) {
   if (0 == strlen(compress)) return true;
@@ -267,7 +267,8 @@ bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRES
     strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
     return true;
   }
-  return checkColumnCompress(compress);
+
+  return checkColumnCompress(compress) && validColCompress(type, columnCompressVal(compress));
 }
 bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) {
   if (0 == strlen(level)) return true;
@@ -288,7 +289,7 @@ bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTI
     strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
     return true;
   }
-  return checkColumnLevel(level);
+  return checkColumnLevel(level) && validColCompressLevel(type, columnLevelVal(level));
 }
 
 void setColEncode(uint32_t* compress, uint8_t l1) {

source/common/src/tdatablock.c

@@ -2411,13 +2411,24 @@ _end:
 
 void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId) {
   char tmp[TSDB_TABLE_NAME_LEN] = {0};
-  if (stbName == NULL) {
+  if (stbName == NULL){
-    snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%" PRIu64, groupId);
+    snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%"PRIu64, groupId);
-  } else {
+  }else{
-    snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%" PRIu64, stbName, groupId);
+    int32_t i = strlen(stbName) - 1;
+    for(; i >= 0; i--){
+      if (stbName[i] == '.'){
+        break;
+      }
+    }
+    snprintf(tmp, TSDB_TABLE_NAME_LEN, "_%s_%"PRIu64, stbName + i + 1, groupId);
   }
   ctbName[TSDB_TABLE_NAME_LEN - strlen(tmp) - 1] = 0;  // put stbname + groupId to the end
   strcat(ctbName, tmp);
+  for(int i = 0; i < strlen(ctbName); i++){
+    if(ctbName[i] == '.'){
+      ctbName[i] = '_';
+    }
+  }
 }
 
 // auto stream subtable name starts with 't_', followed by the first segment of MD5 digest for group vals.

source/common/src/tdataformat.c

@@ -1183,8 +1183,7 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in
   }
 }
 
-void tRowGetKey(SRow *row, SRowKey *key) {
+void tRowGetPrimaryKey(SRow *row, SRowKey *key) {
-  key->ts = row->ts;
   key->numOfPKs = row->numOfPKs;
 
   if (key->numOfPKs == 0) {
@@ -1283,10 +1282,7 @@ int32_t tValueCompare(const SValue *tv1, const SValue *tv2) {
 // NOTE:
 // set key->numOfPKs to 0 as the smallest key with ts
 // set key->numOfPKs to (TD_MAX_PK_COLS + 1) as the largest key with ts
-int32_t tRowKeyCompare(const void *p1, const void *p2) {
+FORCE_INLINE int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2) {
-  SRowKey *key1 = (SRowKey *)p1;
-  SRowKey *key2 = (SRowKey *)p2;
-
   if (key1->ts < key2->ts) {
     return -1;
   } else if (key1->ts > key2->ts) {

source/common/src/tglobal.c

@@ -32,22 +32,22 @@
 SConfig *tsCfg = NULL;
 
 // cluster
-char     tsFirst[TSDB_EP_LEN] = {0};
+char          tsFirst[TSDB_EP_LEN] = {0};
-char     tsSecond[TSDB_EP_LEN] = {0};
+char          tsSecond[TSDB_EP_LEN] = {0};
-char     tsLocalFqdn[TSDB_FQDN_LEN] = {0};
+char          tsLocalFqdn[TSDB_FQDN_LEN] = {0};
-char     tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
+char          tsLocalEp[TSDB_EP_LEN] = {0};  // Local End Point, hostname:port
-char     tsVersionName[16] = "community";
+char          tsVersionName[16] = "community";
-uint16_t tsServerPort = 6030;
+uint16_t      tsServerPort = 6030;
-int32_t  tsVersion = 30000000;
+int32_t       tsVersion = 30000000;
-int32_t  tsStatusInterval = 1;  // second
+int32_t       tsStatusInterval = 1;  // second
-int32_t  tsNumOfSupportVnodes = 256;
+int32_t       tsNumOfSupportVnodes = 256;
-char     tsEncryptAlgorithm[16] = {0};
+char          tsEncryptAlgorithm[16] = {0};
-char     tsEncryptScope[100] = {0};
+char          tsEncryptScope[100] = {0};
-EEncryptAlgor  tsiEncryptAlgorithm = 0;
+EEncryptAlgor tsiEncryptAlgorithm = 0;
-EEncryptScope  tsiEncryptScope = 0;
+EEncryptScope tsiEncryptScope = 0;
-//char     tsAuthCode[500] = {0};
+// char     tsAuthCode[500] = {0};
-//char     tsEncryptKey[17] = {0};
+// char     tsEncryptKey[17] = {0};
-char     tsEncryptKey[17] = {0};
+char tsEncryptKey[17] = {0};
 
 // common
 int32_t tsMaxShellConns = 50000;
@@ -240,8 +240,8 @@ int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
 int64_t tsTickPerHour[] = {3600000L, 3600000000L, 3600000000000L};
 
 // lossy compress 7
-char tsLossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty
+char tsLossyColumns[32] = "float|double";  // "float|double" means all float and double columns can be lossy compressed.
-                               // can close lossy compress.
+                                           // set empty can close lossy compress.
 // below option can take effect when tsLossyColumns not empty
 float    tsFPrecision = 1E-8;                   // float column precision
 double   tsDPrecision = 1E-16;                  // double column precision
@@ -296,6 +296,8 @@ char   tsS3AccessKeySecret[TSDB_FQDN_LEN] = "<accesskeysecrect>";
 char   tsS3BucketName[TSDB_FQDN_LEN] = "<bucketname>";
 char   tsS3AppId[TSDB_FQDN_LEN] = "<appid>";
 int8_t tsS3Enabled = false;
+int8_t tsS3EnabledCfg = false;
+int8_t tsS3Oss = false;
 int8_t tsS3StreamEnabled = false;
 
 int8_t tsS3Https = true;
@@ -310,7 +312,7 @@ bool tsExperimental = true;
 
 int32_t tsMaxTsmaNum = 3;
 int32_t tsMaxTsmaCalcDelay = 600;
-int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d
+int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24;  // in ms, default to 1d
 
 #ifndef _STORAGE
 int32_t taosSetTfsCfg(SConfig *pCfg) {
@@ -366,10 +368,15 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
       tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN);
     }
   }
+  char *oss = strstr(tsS3Endpoint, "aliyuncs.");
+  if (oss) {
+    tsS3Oss = true;
+  }
   if (tsS3BucketName[0] != '<') {
 #if defined(USE_COS) || defined(USE_S3)
 #ifdef TD_ENTERPRISE
     /*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
+    tsS3EnabledCfg = true;
 #endif
     tsS3StreamEnabled = true;
 #endif
@@ -563,11 +570,12 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
   if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
   if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
+  if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT,
+                 CFG_DYN_CLIENT) != 0)
+    return -1;
   if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
     return -1;
-  if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
+  if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
-      0)
     return -1;
   if (cfgAddInt32(pCfg, "tsmaDataDeleteMark", tsmaDataDeleteMark, 60 * 60 * 1000, INT64_MAX, CFG_SCOPE_CLIENT,
                   CFG_DYN_CLIENT) != 0)
@@ -618,11 +626,12 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   tsNumOfSupportVnodes = tsNumOfCores * 2;
   tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2);
-  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
+  if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
+    return -1;
 
   if (cfgAddString(pCfg, "encryptAlgorithm", tsEncryptAlgorithm, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddString(pCfg, "encryptScope", tsEncryptScope, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
-  //if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
+  // if (cfgAddString(pCfg, "authCode", tsAuthCode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
 
   if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
@@ -749,8 +758,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
     return -1;
 
-  if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
+  if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
-    return -1;
 
   if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 3, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
@@ -807,7 +815,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
 
-  if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
   if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@@ -1181,7 +1188,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
   tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
   tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
-  //tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
+  // tstrncpy(tsAuthCode, cfgGetItem(pCfg, "authCode")->str, 100);
 
   tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
   tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32;
@@ -1268,7 +1275,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
 
   tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
 
-  tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
   tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
   tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
   tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
@@ -1388,7 +1394,7 @@ static int32_t taosCheckGlobalCfg() {
 }
 
 int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
-                    bool tsc) {
+                    bool tsc, bool isDumpCfg) {
   if (tsCfg != NULL) return 0;
   tsCfg = cfgInit();
 
@@ -1435,7 +1441,7 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
 
   taosSetAllDebugFlag(tsCfg, cfgGetItem(tsCfg, "debugFlag")->i32);
 
-  cfgDumpCfg(tsCfg, tsc, false);
+  if(isDumpCfg) cfgDumpCfg(tsCfg, tsc, false);
 
   if (taosCheckGlobalCfg() != 0) {
     return -1;

source/common/src/tmisce.c

@@ -228,6 +228,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
   blockDataEnsureCapacity(pBlock, cfgGetSize(pConf));
   SConfigIter* pIter = cfgCreateIter(pConf);
 
+  cfgLock(pConf);
+
   while ((pItem = cfgNextIter(pIter)) != NULL) {
     col = startCol;
 
@@ -253,6 +255,8 @@ int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol) {
     numOfRows++;
   }
 
+  cfgUnLock(pConf);
+
   pBlock->info.rows = numOfRows;
 
   cfgDestroyIter(pIter);

source/common/src/tmsg.c

@@ -5962,9 +5962,11 @@ int32_t tSerializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceVgr
   tEncoderInit(&encoder, buf, bufLen);
 
   if (tStartEncode(&encoder) < 0) return -1;
-  if (tEncodeI32(&encoder, pReq->useless) < 0) return -1;
+  if (tEncodeI32(&encoder, pReq->reserved) < 0) return -1;
   if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1;
   ENCODESQL();
+  if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
+
   tEndEncode(&encoder);
 
   int32_t tlen = encoder.pos;
@@ -5977,12 +5979,15 @@ int32_t tDeserializeSBalanceVgroupLeaderReq(void *buf, int32_t bufLen, SBalanceV
   tDecoderInit(&decoder, buf, bufLen);
 
   if (tStartDecode(&decoder) < 0) return -1;
-  if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1;
+  if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1;
   if (!tDecodeIsEnd(&decoder)) {
     if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1;
   }
-
   DECODESQL();
+  if (!tDecodeIsEnd(&decoder)) {
+    if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1;
+  }
+
   tEndDecode(&decoder);
 
   tDecoderClear(&decoder);

source/common/test/dataformatTest.cpp

@@ -20,6 +20,7 @@
 #include <tglobal.h>
 #include <tmsg.h>
 #include <iostream>
+#include <tdatablock.h>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wwrite-strings"
@@ -475,6 +476,45 @@ TEST(testCase, AllNormTest) {
   taosMemoryFree(pTSchema);
 }
 
+TEST(testCase, StreamAllNormTest) {
+  char ctbName[TSDB_TABLE_NAME_LEN] = {0};
+  uint64_t groupId = 12345;
+
+  buildCtbNameAddGroupId(NULL, ctbName, groupId);
+
+  ASSERT_STREQ("_12345", ctbName);
+}
+
+TEST(testCase, StreamWithStbName) {
+  char stbName[] = "1.table.stb";
+  char ctbName[TSDB_TABLE_NAME_LEN] = {0};
+  uint64_t groupId = 12345;
+
+  buildCtbNameAddGroupId(stbName, ctbName, groupId);
+
+  ASSERT_STREQ("_stb_12345", ctbName);
+}
+
+TEST(testCase, StreamWithoutDotInStbName) {
+  char stbName[] = "table";
+  char ctbName[TSDB_TABLE_NAME_LEN] = {0};
+  uint64_t groupId = 12345;
+
+  buildCtbNameAddGroupId(stbName, ctbName, groupId);
+
+  ASSERT_STREQ("_table_12345", ctbName);
+}
+
+TEST(testCase, StreamWithoutDotInStbName2) {
+  char stbName[] = "";
+  char ctbName[TSDB_TABLE_NAME_LEN] = {0};
+  uint64_t groupId = 12345;
+
+  buildCtbNameAddGroupId(stbName, ctbName, groupId);
+
+  ASSERT_STREQ("__12345", ctbName);
+}
+
 #if 1
 TEST(testCase, NoneTest) {
   const static int nCols = 14;

source/dnode/mgmt/exe/dmMain.c

@@ -205,11 +205,11 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
       if(i < argc - 1) {
         int32_t len = strlen(argv[++i]);
         if (len < ENCRYPT_KEY_LEN_MIN) {
-          printf("encrypt key is too short, it should be great or equal to %d\n", ENCRYPT_KEY_LEN_MIN);
+          printf("Error: Encrypt key should be at least %d characters\n", ENCRYPT_KEY_LEN_MIN);
           return -1;
         }
         if (len > ENCRYPT_KEY_LEN) {
-          printf("encrypt key overflow, it should be less or equal to %d\n", ENCRYPT_KEY_LEN);
+          printf("Error: Encrypt key overflow, it should be at most %d characters\n", ENCRYPT_KEY_LEN);
           return -1;
         }
         tstrncpy(global.encryptKey, argv[i], ENCRYPT_KEY_LEN);
@@ -325,7 +325,7 @@ int main(int argc, char const *argv[]) {
   }
 
   if (dmParseArgs(argc, argv) != 0) {
-    printf("failed to start since parse args error\n");
+    //printf("failed to start since parse args error\n");
     taosCleanupArgs();
     return -1;
   }
@@ -380,7 +380,11 @@ int mainWindows(int argc, char **argv) {
 
   dmPrintArgs(argc, argv);
 
-  if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
+  bool isDumpCfg = true;
+  if(global.generateCode) {
+    isDumpCfg = false;
+  }
+  if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0, isDumpCfg) != 0) {
     dError("failed to start since read config error");
     taosCloseLog();
     taosCleanupArgs();

source/dnode/mgmt/mgmt_snode/src/smWorker.c

@@ -37,7 +37,7 @@ static void smProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     dTrace("msg:%p, get from snode-write queue", pMsg);
     int32_t code = sndProcessWriteMsg(pMgmt->pSnode, pMsg, NULL);
     if (code < 0) {
-      dGError("snd, msg:%p failed to process write since %s", pMsg, terrstr(code));
+      dGError("snd, msg:%p failed to process write since %s", pMsg, tstrerror(code));
       if (pMsg->info.handle != NULL) {
         tmsgSendRsp(pMsg);
       }
@@ -58,7 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   dTrace("msg:%p, get from snode-stream queue", pMsg);
   int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
   if (code < 0) {
-    dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
+    dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), tstrerror(code));
     smSendRsp(pMsg, terrno);
   }
 

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -316,7 +316,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   if (vmTsmaAdjustDays(&vnodeCfg, &req) < 0) {
     dError("vgId:%d, failed to adjust tsma days since %s", req.vgId, terrstr());
-    code = terrno;
+    code = terrno != 0 ? terrno : -1;
     goto _OVER;
   }
 
@@ -344,21 +344,21 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
     dError("vgId:%d, failed to create vnode since %s", req.vgId, terrstr());
     vmReleaseVnode(pMgmt, pVnode);
     tFreeSCreateVnodeReq(&req);
-    code = terrno;
+    code = terrno != 0 ? terrno : -1;
     return code;
   }
 
   SVnode *pImpl = vnodeOpen(path, diskPrimary, pMgmt->pTfs, pMgmt->msgCb, true);
   if (pImpl == NULL) {
     dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
-    code = terrno;
+    code = terrno != 0 ? terrno : -1;
     goto _OVER;
   }
 
   code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl);
   if (code != 0) {
     dError("vgId:%d, failed to open vnode since %s", req.vgId, terrstr());
-    code = terrno;
+    code = terrno != 0 ? terrno : code;
     goto _OVER;
   }
 
@@ -379,7 +379,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
 
   code = vmWriteVnodeListToFile(pMgmt);
   if (code != 0) {
-    code = terrno;
+    code = terrno != 0 ? terrno : code;
     goto _OVER;
   }
 

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -103,7 +103,7 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
   if (code != 0) {
     if (terrno != 0) code = terrno;
     dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
-            terrstr(code));
+            tstrerror(code));
     vmSendRsp(pMsg, code);
   }
 
@@ -201,7 +201,7 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
     case QUERY_QUEUE:
       code = vnodePreprocessQueryMsg(pVnode->pImpl, pMsg);
       if (code) {
-        dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, terrstr(code));
+        dError("vgId:%d, msg:%p preprocess query msg failed since %s", pVnode->vgId, pMsg, tstrerror(code));
       } else {
         dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg);
         taosWriteQitem(pVnode->pQueryQ, pMsg);
@@ -219,13 +219,13 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
       if (!vmDataSpaceSufficient(pVnode)) {
         terrno = TSDB_CODE_NO_ENOUGH_DISKSPACE;
         code = terrno;
-        dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
+        dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
         break;
       }
       if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) {
         terrno = TSDB_CODE_VND_NO_WRITE_AUTH;
         code = terrno;
-        dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code));
+        dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, tstrerror(code));
         break;
       }
       if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) {

source/dnode/mgmt/node_mgmt/src/dmEnv.c

@@ -17,6 +17,7 @@
 #include "dmMgmt.h"
 #include "audit.h"
 #include "libs/function/tudf.h"
+#include "tgrant.h"
 
 #define DM_INIT_AUDIT()              \
   do {                               \
@@ -150,6 +151,7 @@ static bool dmCheckDataDirVersion() {
 
 extern int32_t s3Begin();
 extern void    s3End();
+extern int8_t  tsS3Enabled;
 
 #endif
 

source/dnode/mgmt/node_mgmt/src/dmMgmt.c

@@ -18,12 +18,10 @@
 #include "dmNodes.h"
 #include "index.h"
 #include "qworker.h"
-#include "tstream.h"
-#ifdef TD_TSZ
 #include "tcompression.h"
 #include "tglobal.h"
 #include "tgrant.h"
-#endif
+#include "tstream.h"
 
 static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
   SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@@ -48,10 +46,8 @@ int32_t dmInitDnode(SDnode *pDnode) {
     goto _OVER;
   }
 
-#ifdef TD_TSZ
   // compress module init
   tsCompressInit(tsLossyColumns, tsFPrecision, tsDPrecision, tsMaxRange, tsCurRange, (int)tsIfAdtFse, tsCompressor);
-#endif
 
   pDnode->wrappers[DNODE].func = dmGetMgmtFunc();
   pDnode->wrappers[MNODE].func = mmGetMgmtFunc();
@@ -88,7 +84,7 @@ int32_t dmInitDnode(SDnode *pDnode) {
   streamMetaInit();
 
   dmInitStatusClient(pDnode);
-  dmInitSyncClient(pDnode);  
+  dmInitSyncClient(pDnode);
 
   dmReportStartup("dnode-transport", "initialized");
   dDebug("dnode is created, ptr:%p", pDnode);
@@ -120,10 +116,8 @@ void dmCleanupDnode(SDnode *pDnode) {
   indexCleanup();
   taosConvDestroy();
 
-#ifdef TD_TSZ
   // compress destroy
   tsCompressExit();
-#endif
 
   dDebug("dnode is closed, ptr:%p", pDnode);
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -821,6 +821,10 @@ static int32_t mndCheckDbEncryptKey(SMnode *pMnode, SCreateDbReq *pReq) {
 
 #ifdef TD_ENTERPRISE
   if (pReq->encryptAlgorithm == TSDB_ENCRYPT_ALGO_NONE) goto _exit;
+  if (grantCheck(TSDB_GRANT_DB_ENCRYPTION) != 0) {
+    code = TSDB_CODE_MND_DB_ENCRYPT_GRANT_EXPIRED;
+    goto _exit;
+  }
   if (tsEncryptionKeyStat != ENCRYPT_KEY_STAT_LOADED) {
     code = TSDB_CODE_MND_INVALID_ENCRYPT_KEY;
     mError("db:%s, failed to check encryption key:%" PRIi8 " in mnode leader since it's not loaded", pReq->db,
@@ -903,6 +907,13 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (createReq.replications == 2) {
+    if ((terrno = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
+      code = terrno;
+      goto _OVER;
+    }
+  }
+
   if ((code = mndCheckDbEncryptKey(pMnode, &createReq)) != 0) {
     terrno = code;
     goto _OVER;
@@ -1163,6 +1174,12 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) {
     goto _OVER;
   }
 
+  if (alterReq.replications == 2) {
+    if ((code = grantCheck(TSDB_GRANT_DUAL_REPLICA_HA)) != 0) {
+      goto _OVER;
+    }
+  }
+
   int32_t numOfTopics = 0;
   if (mndGetNumOfTopics(pMnode, pDb->name, &numOfTopics) != 0) {
     goto _OVER;

source/dnode/vnode/CMakeLists.txt

@@ -1,6 +1,9 @@
 # vnode
 add_subdirectory(src/tqCommon)
 add_library(vnode STATIC "")
+if(${TD_DARWIN})
+    target_compile_options(vnode PRIVATE -Wno-error=single-bit-bitfield-constant-conversion)
+endif(${TD_DARWIN})
 set(
   VNODE_SOURCE_FILES
   "src/vnd/vnodeOpen.c"

source/dnode/vnode/src/inc/tsdb.h

@@ -120,12 +120,32 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) {
 #define tsdbRowFromBlockData(BLOCKDATA, IROW) \
   ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)})
 
+#define TSDBROW_INIT_KEY(_ROW, _KEY)                             \
+  {                                                              \
+    if ((_ROW)->type == TSDBROW_ROW_FMT) {                       \
+      _KEY.version = (_ROW)->version;                            \
+      _KEY.ts = (_ROW)->pTSRow->ts;                              \
+    } else {                                                     \
+      _KEY.version = (_ROW)->pBlockData->aVersion[(_ROW)->iRow]; \
+      _KEY.ts = (_ROW)->pBlockData->aTSKEY[(_ROW)->iRow];        \
+    }                                                            \
+  }
+
+#define tColRowGetKey(_pBlock, _irow, _key)             \
+  {                                                     \
+    (_key)->ts = (_pBlock)->aTSKEY[(_irow)];            \
+    (_key)->numOfPKs = 0;                               \
+    if ((_pBlock)->nColData > 0) {                      \
+      tColRowGetPrimaryKey((_pBlock), (_irow), (_key)); \
+    }                                                   \
+  }
+
 void    tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal);
 int32_t tsdbRowCompare(const void *p1, const void *p2);
 int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
 int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
 void    tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
-void    tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
+void    tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
 
 
 // STSDBRowIter
@@ -902,9 +922,16 @@ typedef struct {
   SColVal colVal;
 } SLastCol;
 
+typedef struct {
+  int8_t      lflag;
+  STsdbRowKey tsdbRowKey;
+  SColVal     colVal;
+} SLastUpdateCtx;
+
 int32_t tsdbOpenCache(STsdb *pTsdb);
 void    tsdbCloseCache(STsdb *pTsdb);
-int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *row);
+int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow);
+int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData);
 int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 
 int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb);
@@ -946,7 +973,28 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
 // #define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
 // #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
 
-TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter);
+static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
+  if (pIter == NULL) return NULL;
+
+  if (pIter->pRow) {
+    return pIter->pRow;
+  }
+
+  if (pIter->backward) {
+    if (pIter->pNode == pIter->pTbData->sl.pHead) {
+      return NULL;
+    }
+  } else {
+    if (pIter->pNode == pIter->pTbData->sl.pTail) {
+      return NULL;
+    }
+  }
+
+  pIter->pRow = &pIter->row;
+  pIter->row = pIter->pNode->row;
+
+  return pIter->pRow;
+}
 
 typedef struct {
   int64_t  suid;

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -16,6 +16,7 @@
 #include "functionMgt.h"
 #include "tsdb.h"
 #include "tsdbDataFileRW.h"
+#include "tsdbIter.h"
 #include "tsdbReadUtil.h"
 #include "vnd.h"
 
@@ -359,7 +360,10 @@ static int32_t tsdbCacheDeserializeV0(char const *value, SLastCol *pLastCol) {
 
   if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
     pLastCol->colVal.value.nData = pLastColV0->colVal.value.nData;
-    pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
+    pLastCol->colVal.value.pData = NULL;
+    if (pLastCol->colVal.value.nData > 0) {
+      pLastCol->colVal.value.pData = (uint8_t *)(&pLastColV0[1]);
+    }
     return sizeof(SLastColV0) + pLastColV0->colVal.value.nData;
   } else {
     pLastCol->colVal.value.val = pLastColV0->colVal.value.val;
@@ -372,7 +376,7 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
     return NULL;
   }
 
-  SLastCol* pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
+  SLastCol *pLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
   if (NULL == pLastCol) {
     return NULL;
   }
@@ -401,8 +405,11 @@ static SLastCol *tsdbCacheDeserialize(char const *value, size_t size) {
     offset += sizeof(SValue);
 
     if (IS_VAR_DATA_TYPE(pLastCol->rowKey.pks[i].type)) {
-      pLastCol->rowKey.pks[i].pData = (uint8_t *)value + offset;
+      pLastCol->rowKey.pks[i].pData = NULL;
-      offset += pLastCol->rowKey.pks[i].nData;
+      if (pLastCol->rowKey.pks[i].nData > 0) {
+        pLastCol->rowKey.pks[i].pData = (uint8_t *)value + offset;
+        offset += pLastCol->rowKey.pks[i].nData;
+      }
     }
   }
 
@@ -940,10 +947,37 @@ typedef struct {
   SLastKey key;
 } SIdxKey;
 
-static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
+static int32_t tsdbCacheUpdateValue(SValue *pOld, SValue *pNew) {
-  uint8_t *pVal = NULL;
+  uint8_t *pFree = NULL;
   int      nData = 0;
 
+  if (IS_VAR_DATA_TYPE(pOld->type)) {
+    pFree = pOld->pData;
+    nData = pOld->nData;
+  }
+
+  *pOld = *pNew;
+  if (IS_VAR_DATA_TYPE(pNew->type)) {
+    if (nData < pNew->nData) {
+      pOld->pData = taosMemoryCalloc(1, pNew->nData);
+    } else {
+      pOld->pData = pFree;
+      pFree = NULL;
+    }
+
+    if (pNew->nData) {
+      memcpy(pOld->pData, pNew->pData, pNew->nData);
+    } else {
+      pFree = pOld->pData;
+      pOld->pData = NULL;
+    }
+  }
+
+  taosMemoryFreeClear(pFree);
+  return 0;
+}
+
+static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal *pColVal) {
   // update rowkey
   pLastCol->rowKey.ts = pRowKey->ts;
   pLastCol->rowKey.numOfPKs = pRowKey->numOfPKs;
@@ -951,87 +985,43 @@ static void tsdbCacheUpdateLastCol(SLastCol *pLastCol, SRowKey *pRowKey, SColVal
     SValue *pPKValue = &pLastCol->rowKey.pks[i];
     SValue *pNewPKValue = &pRowKey->pks[i];
 
-    if (IS_VAR_DATA_TYPE(pPKValue->type)) {
+    (void)tsdbCacheUpdateValue(pPKValue, pNewPKValue);
-      pVal = pPKValue->pData;
-      nData = pPKValue->nData;
-    }
-    *pPKValue = *pNewPKValue;
-    if (IS_VAR_DATA_TYPE(pPKValue->type)) {
-      if (nData < pPKValue->nData) {
-        taosMemoryFree(pVal);
-        pPKValue->pData = taosMemoryCalloc(1, pNewPKValue->nData);
-      } else {
-        pPKValue->pData = pVal;
-      }
-      if (pNewPKValue->nData) {
-        memcpy(pPKValue->pData, pNewPKValue->pData, pNewPKValue->nData);
-      }
-    }
   }
 
   // update colval
-  if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
+  pLastCol->colVal.cid = pColVal->cid;
-    nData = pLastCol->colVal.value.nData;
+  pLastCol->colVal.flag = pColVal->flag;
-    pVal = pLastCol->colVal.value.pData;
+  (void)tsdbCacheUpdateValue(&pLastCol->colVal.value, &pColVal->value);
-  }
-  pLastCol->colVal = *pColVal;
-  if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
-    if (nData < pColVal->value.nData) {
-      taosMemoryFree(pVal);
-      pLastCol->colVal.value.pData = taosMemoryCalloc(1, pColVal->value.nData);
-    } else {
-      pLastCol->colVal.value.pData = pVal;
-    }
-    if (pColVal->value.nData) {
-      memcpy(pLastCol->colVal.value.pData, pColVal->value.pData, pColVal->value.nData);
-    }
-  }
 
   if (!pLastCol->dirty) {
     pLastCol->dirty = 1;
   }
 }
 
-int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow) {
+static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) {
+  if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) {
+    return 0;
+  }
+
   int32_t code = 0;
 
-  // 1, fetch schema
+  int        num_keys = TARRAY_SIZE(updCtxArray);
-  STSchema *pTSchema = NULL;
-  int32_t   sver = TSDBROW_SVERSION(pRow);
-
-  code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    return -1;
-  }
-
-  // 2, iterate col values into array
-  SArray *aColVal = taosArrayInit(32, sizeof(SColVal));
-
-  STSDBRowIter iter = {0};
-  tsdbRowIterOpen(&iter, pRow, pTSchema);
-
-  for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
-    taosArrayPush(aColVal, pColVal);
-  }
-
-  tsdbRowClose(&iter);
-
-  // 3, build keys & multi get from rocks
-  int        num_keys = TARRAY_SIZE(aColVal);
   SArray    *remainCols = NULL;
   SLRUCache *pCache = pTsdb->lruCache;
 
-  STsdbRowKey tsdbRowKey = {0};
-  tsdbRowGetKey(pRow, &tsdbRowKey);
-  SRowKey *pRowKey = &tsdbRowKey.key;
-
   taosThreadMutexLock(&pTsdb->lruMutex);
   for (int i = 0; i < num_keys; ++i) {
-    SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, i);
+    SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
-    int16_t  cid = pColVal->cid;
 
-    SLastKey  *key = &(SLastKey){.lflag = LFLAG_LAST_ROW, .uid = uid, .cid = cid};
+    int8_t   lflag = updCtx->lflag;
+    SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
+    SColVal *pColVal = &updCtx->colVal;
+
+    if (lflag == LFLAG_LAST && !COL_VAL_IS_VALUE(pColVal)) {
+      continue;
+    }
+
+    SLastKey  *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid};
     size_t     klen = ROCKS_KEY_LEN;
     LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
     if (h) {
@@ -1047,23 +1037,6 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
       }
       taosArrayPush(remainCols, &(SIdxKey){i, *key});
     }
-
-    if (COL_VAL_IS_VALUE(pColVal)) {
-      key->lflag = LFLAG_LAST;
-      LRUHandle *h = taosLRUCacheLookup(pCache, key, klen);
-      if (h) {
-        SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
-        if (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1) {
-          tsdbCacheUpdateLastCol(pLastCol, pRowKey, pColVal);
-        }
-        taosLRUCacheRelease(pCache, h, false);
-      } else {
-        if (!remainCols) {
-          remainCols = taosArrayInit(num_keys * 2, sizeof(SIdxKey));
-        }
-        taosArrayPush(remainCols, &(SIdxKey){i, *key});
-      }
-    }
   }
 
   if (remainCols) {
@@ -1091,12 +1064,17 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
     rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
     for (int i = 0; i < num_keys; ++i) {
       SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i];
-      SColVal *pColVal = (SColVal *)TARRAY_DATA(aColVal) + idxKey->idx;
+      SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
-      // SColVal *pColVal = (SColVal *)taosArrayGet(aColVal, idxKey->idx);
+      SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
+      SColVal *pColVal = &updCtx->colVal;
 
       SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]);
       SLastCol *PToFree = pLastCol;
 
+      if (IS_LAST_KEY(idxKey->key) && !COL_VAL_IS_VALUE(pColVal)) {
+        continue;
+      }
+
       if (IS_LAST_ROW_KEY(idxKey->key)) {
         int32_t cmp_res = 1;
         if (pLastCol) {
@@ -1104,8 +1082,8 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
         }
 
         if (NULL == pLastCol || cmp_res < 0 || (cmp_res == 0 && !COL_VAL_IS_NONE(pColVal))) {
-          char  *value = NULL;
+          char    *value = NULL;
-          size_t vlen = 0;
+          size_t   vlen = 0;
           SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
           tsdbCacheSerialize(&lastColTmp, &value, &vlen);
 
@@ -1142,48 +1120,6 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
 
           taosMemoryFree(value);
         }
-      } else {
-        if (COL_VAL_IS_VALUE(pColVal)) {
-          if (NULL == pLastCol || (tRowKeyCompare(&pLastCol->rowKey, pRowKey) != 1)) {
-            char  *value = NULL;
-            size_t vlen = 0;
-            SLastCol lastColTmp = {.rowKey = *pRowKey, .colVal = *pColVal};
-            tsdbCacheSerialize(&lastColTmp, &value, &vlen);
-
-            taosThreadMutexLock(&pTsdb->rCache.rMutex);
-
-            rocksdb_writebatch_put(wb, (char *)&idxKey->key, ROCKS_KEY_LEN, value, vlen);
-
-            taosThreadMutexUnlock(&pTsdb->rCache.rMutex);
-
-            pLastCol = &lastColTmp;
-            SLastCol *pTmpLastCol = taosMemoryCalloc(1, sizeof(SLastCol));
-            *pTmpLastCol = *pLastCol;
-            pLastCol = pTmpLastCol;
-
-            size_t charge = sizeof(*pLastCol);
-            for (int8_t i = 0; i < pLastCol->rowKey.numOfPKs; i++) {
-              SValue *pValue = &pLastCol->rowKey.pks[i];
-              if (IS_VAR_DATA_TYPE(pValue->type)) {
-                reallocVarDataVal(pValue);
-                charge += pValue->nData;
-              }
-            }
-
-            if (IS_VAR_DATA_TYPE(pLastCol->colVal.value.type)) {
-              reallocVarData(&pLastCol->colVal);
-              charge += pLastCol->colVal.value.nData;
-            }
-
-            LRUStatus status = taosLRUCacheInsert(pTsdb->lruCache, &idxKey->key, ROCKS_KEY_LEN, pLastCol, charge,
-                                                  tsdbCacheDeleter, NULL, TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
-            if (status != TAOS_LRU_STATUS_OK) {
-              code = -1;
-            }
-
-            taosMemoryFree(value);
-          }
-        }
       }
 
       taosMemoryFreeClear(PToFree);
@@ -1203,11 +1139,152 @@ int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSDBROW *pRow
   taosThreadMutexUnlock(&pTsdb->lruMutex);
 
 _exit:
-  taosArrayDestroy(aColVal);
-  taosMemoryFree(pTSchema);
   return code;
 }
 
+int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow,
+                                 SRow **aRow) {
+  int32_t code = 0;
+
+  // 1. prepare last
+  TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version};
+
+  STSchema *pTSchema = NULL;
+  int32_t   sver = TSDBROW_SVERSION(&lRow);
+  SArray   *ctxArray = NULL;
+  SSHashObj *iColHash = NULL;
+
+  code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    goto _exit;
+  }
+
+  TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version};
+  int32_t nCol = pTSchema->numOfCols;
+
+  ctxArray = taosArrayInit(nCol, sizeof(SLastUpdateCtx));
+  iColHash = tSimpleHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT));
+
+  // 1. prepare by lrow
+  STsdbRowKey tsdbRowKey = {0};
+  tsdbRowGetKey(&lRow, &tsdbRowKey);
+
+  STSDBRowIter iter = {0};
+  tsdbRowIterOpen(&iter, &lRow, pTSchema);
+  int32_t iCol = 0;
+  for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal && iCol < nCol; pColVal = tsdbRowIterNext(&iter), iCol++) {
+    SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
+    taosArrayPush(ctxArray, &updateCtx);
+
+    if (!COL_VAL_IS_VALUE(pColVal)) {
+      tSimpleHashPut(iColHash, &iCol, sizeof(iCol), NULL, 0);
+      continue;
+    }
+    updateCtx.lflag = LFLAG_LAST;
+    taosArrayPush(ctxArray, &updateCtx);
+  }
+  tsdbRowClose(&iter);
+
+  // 2. prepare by the other rows
+  for (int32_t iRow = nRow - 2; iRow >= 0; --iRow) {
+    if (tSimpleHashGetSize(iColHash) == 0) {
+      break;
+    }
+
+    tRow.pTSRow = aRow[iRow];
+
+    STsdbRowKey tsdbRowKey = {0};
+    tsdbRowGetKey(&tRow, &tsdbRowKey);
+
+    void   *pIte = NULL;
+    int32_t iter = 0;
+    while ((pIte = tSimpleHashIterate(iColHash, pIte, &iter)) != NULL) {
+      int32_t iCol = ((int32_t *)pIte)[0];
+      SColVal colVal = COL_VAL_NONE(0, 0);
+      tsdbRowGetColVal(&tRow, pTSchema, iCol, &colVal);
+
+      if (COL_VAL_IS_VALUE(&colVal)) {
+        SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
+        taosArrayPush(ctxArray, &updateCtx);
+        tSimpleHashIterateRemove(iColHash, &iCol, sizeof(iCol), &pIte, &iter);
+      }
+    }
+  }
+
+  // 3. do update
+  tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
+
+_exit:
+  taosMemoryFreeClear(pTSchema);
+  taosArrayDestroy(ctxArray);
+  tSimpleHashCleanup(iColHash);
+  return code;
+}
+
+int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) {
+  int32_t code = 0;
+
+  TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1);
+
+  STSchema *pTSchema = NULL;
+  int32_t   sver = TSDBROW_SVERSION(&lRow);
+  SArray   *ctxArray = NULL;
+
+  code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
+  if (code != TSDB_CODE_SUCCESS) {
+    terrno = code;
+    goto _exit;
+  }
+
+  ctxArray = taosArrayInit(pBlockData->nColData, sizeof(SLastUpdateCtx));
+
+  // 1. prepare last
+  TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0);
+
+  for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) {
+    SColData *pColData = &pBlockData->aColData[iColData];
+    if ((pColData->flag & HAS_VALUE) != HAS_VALUE) {
+      continue;
+    }
+
+    for (tRow.iRow = pBlockData->nRow - 1; tRow.iRow >= 0; --tRow.iRow) {
+      STsdbRowKey tsdbRowKey = {0};
+      tsdbRowGetKey(&tRow, &tsdbRowKey);
+
+      uint8_t colType = tColDataGetBitValue(pColData, tRow.iRow);
+      if (colType == 2) {
+        SColVal colVal = COL_VAL_NONE(pColData->cid, pColData->type);
+        tColDataGetValue(pColData, tRow.iRow, &colVal);
+
+        SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST, .tsdbRowKey = tsdbRowKey, .colVal = colVal};
+        taosArrayPush(ctxArray, &updateCtx);
+        break;
+      }
+    }
+  }
+
+  // 2. prepare last row
+  STsdbRowKey tsdbRowKey = {0};
+  tsdbRowGetKey(&lRow, &tsdbRowKey);
+
+  STSDBRowIter iter = {0};
+  tsdbRowIterOpen(&iter, &lRow, pTSchema);
+  for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) {
+    SLastUpdateCtx updateCtx = {.lflag = LFLAG_LAST_ROW, .tsdbRowKey = tsdbRowKey, .colVal = *pColVal};
+    taosArrayPush(ctxArray, &updateCtx);
+  }
+  tsdbRowClose(&iter);
+
+  // 3. do update
+  tsdbCacheUpdate(pTsdb, suid, uid, ctxArray);
+
+_exit:
+  taosMemoryFreeClear(pTSchema);
+  taosArrayDestroy(ctxArray);
+  return 0;
+}
+
 static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols,
                             int nCols, int16_t *slotIds);
 
@@ -1481,7 +1558,10 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
     taosArraySet(pLastArray, idxKey->idx, pLastCol);
     // taosArrayRemove(remainCols, i);
 
-    if (!pTmpColArray) {
+    if (/*!pTmpColArray*/ lastTmpIndexArray && !lastTmpColArray) {
+      continue;
+    }
+    if (/*!pTmpColArray*/ lastrowTmpIndexArray && lastrowTmpColArray) {
       continue;
     }
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -629,14 +629,12 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
   SMemSkipListNode *pos[SL_MAX_LEVEL];
   TSDBROW           tRow = tsdbRowFromBlockData(pBlockData, 0);
   STsdbRowKey       key;
-  TSDBROW           lRow;  // last row
 
   // first row
   tsdbRowGetKey(&tRow, &key);
   tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
   if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit;
   pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
-  lRow = tRow;
 
   // remain row
   ++tRow.iRow;
@@ -653,7 +651,6 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
       }
 
       if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit;
-      lRow = tRow;
 
       ++tRow.iRow;
     }
@@ -664,7 +661,7 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData,
   }
 
   if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
-    tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
+    tsdbCacheColFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, pBlockData);
   }
 
   // SMemTable
@@ -688,7 +685,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
   SMemSkipListNode *pos[SL_MAX_LEVEL];
   TSDBROW           tRow = {.type = TSDBROW_ROW_FMT, .version = version};
   int32_t           iRow = 0;
-  TSDBROW           lRow;
 
   // backward put first data
   tRow.pTSRow = aRow[iRow++];
@@ -696,7 +692,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
   tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
   code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0);
   if (code) goto _exit;
-  lRow = tRow;
 
   pTbData->minKey = TMIN(pTbData->minKey, key.key.ts);
 
@@ -717,8 +712,6 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
       code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1);
       if (code) goto _exit;
 
-      lRow = tRow;
-
       iRow++;
     }
   }
@@ -727,7 +720,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData,
     pTbData->maxKey = key.key.ts;
   }
   if (!TSDB_CACHE_NO(pMemTable->pTsdb->pVnode->config)) {
-    tsdbCacheUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, &lRow);
+    tsdbCacheRowFormatUpdate(pMemTable->pTsdb, pTbData->suid, pTbData->uid, version, nRow, aRow);
   }
 
   // SMemTable
@@ -806,26 +799,3 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) {
 _exit:
   return aTbDataP;
 }
-
-TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
-  if (pIter == NULL) return NULL;
-
-  if (pIter->pRow) {
-    return pIter->pRow;
-  }
-
-  if (pIter->backward) {
-    if (pIter->pNode == pIter->pTbData->sl.pHead) {
-      return NULL;
-    }
-  } else {
-    if (pIter->pNode == pIter->pTbData->sl.pTail) {
-      return NULL;
-    }
-  }
-
-  pIter->pRow = &pIter->row;
-  pIter->row = pIter->pNode->row;
-
-  return pIter->pRow;
-}

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -828,7 +828,7 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTre
   SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));
   SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));
 
-  SRowKey rkey1, rkey2;
+  SRowKey rkey1 = {0}, rkey2 = {0};
   tRowGetKeyEx(&pIter1->rInfo.row, &rkey1);
   tRowGetKeyEx(&pIter2->rInfo.row, &rkey2);
 

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -24,6 +24,16 @@
 
 #define ASCENDING_TRAVERSE(o)       (o == TSDB_ORDER_ASC)
 #define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))
+#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey)      \
+  do {                                                             \
+    (_pKey)->ts = (_pBlock)->aTSKEY[(_irow)];                      \
+    (_pKey)->numOfPKs = 0;                                         \
+    if ((_slotId) != -1) {                                         \
+      tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \
+    }                                                              \
+  } while (0)
+
+#define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey))
 
 typedef struct {
   bool overlapWithNeighborBlock;
@@ -75,11 +85,9 @@ static void          getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey
 static void          updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);
 static int32_t       buildFromPreFilesetBuffer(STsdbReader* pReader);
 
-static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
-
 static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
 
-int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
+FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
   if (p2 == NULL) {
     return 1;
   }
@@ -101,13 +109,7 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
   }
 }
 
-static void tColRowGetKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
+static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
-  pKey->ts = pBlock->aTSKEY[irow];
-  if (slotId == -1) {
-    pKey->numOfPKs = 0;
-    return;
-  }
-
   SColData* pColData = &pBlock->aColData[slotId];
   SColVal   cv;
   tColDataGetValue(pColData, irow, &cv);
@@ -131,13 +133,7 @@ static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) {
   return n;
 }
 
-static void tRowGetKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
+static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
-  pKey->ts = pRow->ts;
-  pKey->numOfPKs = pRow->numOfPKs;
-  if (pKey->numOfPKs == 0) {
-    return;
-  }
-
   SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
   ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS);
 
@@ -1735,7 +1731,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo*
     pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
   }
 
-  SRowKey k;
+  SRowKey k = {0};
   tRowGetKeyEx(pRow, &k);
 
   STSchema* pSchema = NULL;
@@ -1939,7 +1935,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
     pfKey = NULL;
   }
 
-  SRowKey k, ik;
+  SRowKey k = {0}, ik = {0};
   tRowGetKeyEx(pRow, &k);
   tRowGetKeyEx(piRow, &ik);
 
@@ -2086,7 +2082,7 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan
 }
 
 static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
-  SRowKey rowKey;
+  SRowKey rowKey = {0};
 
   while (1) {
     TSDBROW* pRow = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader);
@@ -3559,7 +3555,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t
   return false;
 }
 
-TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
+FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader) {
   if (!pIter->hasVal) {
     return NULL;
   }
@@ -3567,7 +3563,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
   int32_t  order = pReader->info.order;
   TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter);
 
-  TSDBKEY  key = TSDBROW_KEY(pRow);
+  TSDBKEY  key;
+  TSDBROW_INIT_KEY(pRow, key);
   if (outOfTimeWindow(key.ts, &pReader->info.window)) {
     pIter->hasVal = false;
     return NULL;
@@ -3593,7 +3590,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p
 
     pRow = tsdbTbDataIterGet(pIter->iter);
 
-    key = TSDBROW_KEY(pRow);
+    TSDBROW_INIT_KEY(pRow, key);
     if (outOfTimeWindow(key.ts, &pReader->info.window)) {
       pIter->hasVal = false;
       return NULL;
@@ -3632,10 +3629,12 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArra
       break;
     }
 
-    SRowKey nextKey = {0};
+    if (pCurKey->numOfPKs > 0) {
-    tRowGetKeyEx(pRow, &nextKey);
+      SRowKey nextKey = {0};
-    if (pkCompEx(pCurKey, &nextKey) != 0) {
+      tRowGetKeyEx(pRow, &nextKey);
-      break;
+      if (pkCompEx(pCurKey, &nextKey) != 0) {
+        break;
+      }
     }
 
     STSchema* pTSchema = NULL;
@@ -3785,12 +3784,14 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
         return TSDB_CODE_SUCCESS;
       }
 
-      SRowKey nextRowKey = {0};
+      if (pKey->numOfPKs > 0) {
-      tRowGetKeyEx(pNextRow, &nextRowKey);
+        SRowKey nextRowKey = {0};
-      if (pKey->numOfPKs > 0 && pkCompEx(pKey, &nextRowKey) != 0) {
+        tRowGetKeyEx(pNextRow, &nextRowKey);
-        *pResRow = current;
+        if (pkCompEx(pKey, &nextRowKey) != 0) {
-        *freeTSRow = false;
+          *pResRow = current;
-        return TSDB_CODE_SUCCESS;
+          *freeTSRow = false;
+          return TSDB_CODE_SUCCESS;
+        }
       }
     }
   }
@@ -3801,7 +3802,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
   // start to merge duplicated rows
   STSchema* pTSchema = NULL;
   if (current.type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory
-    pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(&current), pReader, uid);
+    pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid);
     if (pTSchema == NULL) {
       return terrno;
     }
@@ -3814,7 +3815,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIt
 
   STSchema* pTSchema1 = NULL;
   if (pNextRow->type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory
-    pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid);
+    pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid);
     if (pTSchema1 == NULL) {
       return terrno;
     }
@@ -3926,37 +3927,49 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
   // todo refactor
   bool asc = ASCENDING_TRAVERSE(pReader->info.order);
   if (piter->hasVal) {
-    TSDBKEY k = TSDBROW_KEY(pRow);
+    tRowGetKeyEx(pRow, &rowKey);
-    if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
+    if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) {
       pRow = NULL;
     }
   }
 
   if (piiter->hasVal) {
-    TSDBKEY k = TSDBROW_KEY(piRow);
+    tRowGetKeyEx(piRow, &irowKey);
-    if ((k.ts >= endKey && asc) || (k.ts <= endKey && !asc)) {
+    if ((irowKey.ts >= endKey && asc) || (irowKey.ts <= endKey && !asc)) {
       piRow = NULL;
     }
   }
 
-  if (piter->hasVal && piiter->hasVal && pRow != NULL && piRow != NULL) {
+  if (pRow != NULL && piRow != NULL) {
-    tRowGetKeyEx(pRow, &rowKey);
-    tRowGetKeyEx(piRow, &irowKey);
-
     int32_t code = TSDB_CODE_SUCCESS;
-    int32_t ret = pkCompEx(&rowKey, &irowKey);
+    if (rowKey.numOfPKs == 0) {
-    if (ret != 0) {
+      if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) {  // ik.ts < k.ts
-      if ((ret > 0 && asc) || (ret < 0 && (!asc))) {  // ik.ts < k.ts
         code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
-      } else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {
+      } else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) {
         code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
+      } else {  // ik.ts == k.ts
+        *freeTSRow = true;
+        pResRow->type = TSDBROW_ROW_FMT;
+        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
       }
-    } else {  // ik.ts == k.ts
+    } else {
-      *freeTSRow = true;
+      int32_t ret = pkCompEx(&rowKey, &irowKey);
-      pResRow->type = TSDBROW_ROW_FMT;
+      if (ret != 0) {
-      code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
+        if ((ret > 0 && asc) || (ret < 0 && (!asc))) {  // ik.ts < k.ts
-      if (code != TSDB_CODE_SUCCESS) {
+          code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
-        return code;
+        } else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {
+          code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
+        }
+      } else {  // ik.ts == k.ts
+        *freeTSRow = true;
+        pResRow->type = TSDBROW_ROW_FMT;
+        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
+        if (code != TSDB_CODE_SUCCESS) {
+          return code;
+        }
       }
     }
 
@@ -3964,12 +3977,10 @@ static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbRea
   }
 
   if (piter->hasVal && pRow != NULL) {
-    tRowGetKeyEx(pRow, &rowKey);
     return doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
   }
 
   if (piiter->hasVal && piRow != NULL) {
-    tRowGetKeyEx(piRow, &irowKey);
     return doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
   }
 
@@ -4095,7 +4106,11 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e
     if (row.type == TSDBROW_ROW_FMT) {
       code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
       if (code == TSDB_CODE_SUCCESS) {
-        tRowGetKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
+        pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts;
+        pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs;
+        if (row.pTSRow->numOfPKs > 0) {
+          tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
+        }
       }
 
       if (freeTSRow) {

source/dnode/vnode/src/tsdb/tsdbReadUtil.c

@@ -212,7 +212,7 @@ static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader
   int32_t numOfPks = pReader->suppInfo.numOfPks;
   bool    asc = ASCENDING_TRAVERSE(pReader->info.order);
   int8_t  type = pReader->suppInfo.pk.type;
-  int8_t  bytes = pReader->suppInfo.pk.bytes;
+  int32_t bytes = pReader->suppInfo.pk.bytes;
 
   SRowKey* pRowKey = &pScanInfo->lastProcKey;
   if (asc) {
@@ -1056,7 +1056,7 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) {
   const SSttKeyRange* px1 = p1;
   const SSttKeyRange* px2 = p2;
 
-  int32_t ret = tRowKeyCompare(&px1, px2);
+  int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);
   return ret;
 }
 

source/dnode/vnode/src/tsdb/tsdbReadUtil.h

@@ -38,14 +38,18 @@ extern "C" {
     (_k)->ekey.ts = INT64_MIN; \
   } while (0);
 
-#define tRowGetKeyEx(_pRow, _pKey)                                \
+#define tRowGetKeyEx(_pRow, _pKey)                                         \
-  do {                                                            \
+  {                                                                        \
-    if ((_pRow)->type == TSDBROW_ROW_FMT) {                       \
+    if ((_pRow)->type == TSDBROW_ROW_FMT) {                                \
-      tRowGetKey((_pRow)->pTSRow, (_pKey));                       \
+      (_pKey)->ts = (_pRow)->pTSRow->ts;                                   \
-    } else {                                                      \
+      if ((_pRow)->pTSRow->numOfPKs > 0) {                                 \
-      tColRowGetKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey)); \
+        tRowGetPrimaryKey((_pRow)->pTSRow, (_pKey));                       \
-    }                                                             \
+      }                                                                    \
-  } while (0)
+    } else {                                                               \
+      (_pKey)->ts = (_pRow)->pBlockData->aTSKEY[(_pRow)->iRow];            \
+      tColRowGetPrimaryKey((_pRow)->pBlockData, (_pRow)->iRow, (_pKey));   \
+    }                                                                      \
+  }
 
 typedef enum {
   READER_STATUS_SUSPEND = 0x1,

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -757,6 +757,16 @@ _exit:
 int32_t tsdbS3Migrate(STsdb *tsdb, int64_t now, int32_t sync) {
   int32_t code = 0;
 
+  extern int8_t tsS3EnabledCfg;
+
+  int32_t expired = grantCheck(TSDB_GRANT_OBJECT_STORAGE);
+  if (expired && tsS3Enabled) {
+    tsdbWarn("s3 grant expired: %d", expired);
+    tsS3Enabled = false;
+  } else if (!expired && tsS3EnabledCfg) {
+    tsS3Enabled = true;
+  }
+
   if (!tsS3Enabled) {
     return code;
   }

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -601,17 +601,21 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
   STColumn *pTColumn = &pTSchema->columns[iCol];
   SValue    value;
 
-  ASSERT(iCol > 0);
-
   if (pRow->type == TSDBROW_ROW_FMT) {
     tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal);
   } else if (pRow->type == TSDBROW_COL_FMT) {
-    SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
+    if (iCol == 0) {
-
+      *pColVal =
-    if (pColData) {
+          COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID,
-      tColDataGetValue(pColData, pRow->iRow, pColVal);
+                        ((SValue){.type = TSDB_DATA_TYPE_TIMESTAMP, .val = pRow->pBlockData->aTSKEY[pRow->iRow]}));
     } else {
-      *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
+      SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
+
+      if (pColData) {
+        tColDataGetValue(pColData, pRow->iRow, pColVal);
+      } else {
+        *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
+      }
     }
   } else {
     ASSERT(0);
@@ -628,10 +632,7 @@ void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key) {
   }
 }
 
-void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
+void tColRowGetPrimaryKey(SBlockData *pBlock, int32_t irow, SRowKey *key) {
-  key->ts = pBlock->aTSKEY[irow];
-  key->numOfPKs = 0;
-
   for (int32_t i = 0; i < pBlock->nColData; i++) {
     SColData *pColData = &pBlock->aColData[i];
     if (pColData->cflag & COL_IS_KEY) {
@@ -1820,4 +1821,4 @@ uint32_t tsdbCvtTimestampAlg(uint32_t alg) {
   DEFINE_VAR(alg)
 
   return 0;
-}
+}

source/libs/catalog/src/ctgCache.c

@@ -3226,14 +3226,24 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx
     // get tb cache
     pName = taosArrayGet(pList, i);
     pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
-    if (!pTbCache || !pTbCache->pMeta) {
+    if (!pTbCache) {
       ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
       ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
       taosArrayPush(pCtx->pResList, &(SMetaRes){0});
       continue;
     }
+    CTG_LOCK(CTG_READ, &pTbCache->metaLock);
+    if (!pTbCache->pMeta) {
+      CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
+      ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
+      ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
+      taosArrayPush(pCtx->pResList, &(SMetaRes){0});
+      taosHashRelease(dbCache->tbCache, pTbCache);
+      continue;
+    }
     uint64_t suid = pTbCache->pMeta->suid;
     int8_t   tbType = pTbCache->pMeta->tableType;
+    CTG_UNLOCK(CTG_READ, &pTbCache->metaLock);
     taosHashRelease(dbCache->tbCache, pTbCache);
     SName tsmaSourceTbName = *pName;
 
@@ -3444,17 +3454,18 @@ static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const S
 
 int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
   ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname);
-  int32_t code = 0;
+  int32_t             code = 0;
-  SCtgDBCache* pDbCache = NULL;
+  SCtgDBCache        *pDbCache = NULL;
-  SCtgCacheOperation* pOp = NULL;
+  SCtgCacheOperation *pOp = NULL;
-  char dbFName[TSDB_DB_FNAME_LEN];
+  char                dbFName[TSDB_DB_FNAME_LEN];
+  SCtgTSMACache      *pCtgCache = NULL;
   tNameGetFullDbName(pName, dbFName);
   CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache));
   if (NULL == pDbCache || !pDbCache->tsmaCache) {
     goto _return;
   }
 
-  SCtgTSMACache *pCtgCache = taosHashGet(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
+  pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
   if (!pCtgCache) goto _return;
 
   CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
@@ -3471,10 +3482,11 @@ int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
   }
   CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
   CTG_ERR_JRET(ctgEnqueue(pCtg, pOp));
-
+  taosHashRelease(pDbCache->tsmaCache, pCtgCache);
   return TSDB_CODE_SUCCESS;
 
 _return:
+  if (pCtgCache) taosHashRelease(pDbCache->tsmaCache, pCtgCache);
   if (pOp) {
     taosMemoryFree(pOp->data);
     taosMemoryFree(pOp);

source/libs/executor/CMakeLists.txt

@@ -1,6 +1,9 @@
 aux_source_directory(src EXECUTOR_SRC)
 
 add_library(executor STATIC ${EXECUTOR_SRC})
+if(${TD_DARWIN})
+    target_compile_options(executor PRIVATE -Wno-error=deprecated-non-prototype)
+endif(${TD_DARWIN})
 
 target_link_libraries(executor
         PRIVATE os util common function parser planner qcom scalar nodes index wal tdb geometry

source/libs/executor/inc/executorInt.h

@@ -446,17 +446,23 @@ typedef struct STimeWindowAggSupp {
   SColumnInfoData timeWindowData;  // query time window info for scalar function execution.
 } STimeWindowAggSupp;
 
+typedef struct SSteamOpBasicInfo {
+  int32_t primaryPkIndex;
+  bool    updateOperatorInfo;
+} SSteamOpBasicInfo;
+
 typedef struct SStreamScanInfo {
-  SExprInfo*    pPseudoExpr;
+  SSteamOpBasicInfo basic;
-  int32_t       numOfPseudoExpr;
+  SExprInfo*        pPseudoExpr;
-  SExprSupp     tbnameCalSup;
+  int32_t           numOfPseudoExpr;
-  SExprSupp*    pPartTbnameSup;
+  SExprSupp         tbnameCalSup;
-  SExprSupp     tagCalSup;
+  SExprSupp*        pPartTbnameSup;
-  int32_t       primaryTsIndex;  // primary time stamp slot id
+  SExprSupp         tagCalSup;
-  int32_t       primaryKeyIndex;
+  int32_t           primaryTsIndex;  // primary time stamp slot id
-  SReadHandle   readHandle;
+  int32_t           primaryKeyIndex;
-  SInterval     interval;  // if the upstream is an interval operator, the interval info is also kept here.
+  SReadHandle       readHandle;
-  SColMatchInfo matchInfo;
+  SInterval         interval;  // if the upstream is an interval operator, the interval info is also kept here.
+  SColMatchInfo     matchInfo;
 
   SArray*      pBlockLists;  // multiple SSDatablock.
   SSDataBlock* pRes;         // result SSDataBlock
@@ -568,10 +574,6 @@ typedef struct SOpCheckPointInfo {
   SHashObj* children;  // key:child id
 } SOpCheckPointInfo;
 
-typedef struct SSteamOpBasicInfo {
-  int32_t primaryPkIndex;
-} SSteamOpBasicInfo;
-
 typedef struct SStreamIntervalOperatorInfo {
   SOptrBasicInfo      binfo;           // basic info
   SSteamOpBasicInfo   basic;

source/libs/executor/inc/streamexecutorInt.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef STREAM_EXECUTORINT_H
+#define STREAM_EXECUTORINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "executorInt.h"
+
+void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type);
+bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo);
+void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // STREAM_EXECUTORINT_H

source/libs/executor/src/aggregateoperator.c

@@ -484,7 +484,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n
   
   if (!osTempSpaceAvailable()) {
     code = TSDB_CODE_NO_DISKSPACE;
-    qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", terrstr(code), pKey, tsTempDir);
+    qError("Init stream agg supporter failed since %s, key:%s, tempDir:%s", tstrerror(code), pKey, tsTempDir);
     return code;
   }
 

source/libs/executor/src/executor.c

@@ -824,14 +824,17 @@ int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode) {
   qDebug("%s sync killed execTask", GET_TASKID(pTaskInfo));
   setTaskKilled(pTaskInfo, TSDB_CODE_TSC_QUERY_KILLED);
 
-  taosWLockLatch(&pTaskInfo->lock);
+  while(1) {
-  while (qTaskIsExecuting(pTaskInfo)) {
+    taosWLockLatch(&pTaskInfo->lock);
-    taosMsleep(10);
+    if (qTaskIsExecuting(pTaskInfo)) { // let's wait for 100 ms and try again
+      taosWUnLockLatch(&pTaskInfo->lock);
+      taosMsleep(100);
+    } else { // not running now
+      pTaskInfo->code = rspCode;
+      taosWUnLockLatch(&pTaskInfo->lock);
+      return TSDB_CODE_SUCCESS;
+    }
   }
-  pTaskInfo->code = rspCode;
-  taosWUnLockLatch(&pTaskInfo->lock);
-
-  return TSDB_CODE_SUCCESS;
 }
 
 bool qTaskIsExecuting(qTaskInfo_t qinfo) {

source/libs/executor/src/scanoperator.c

@@ -20,6 +20,7 @@
 #include "os.h"
 #include "querynodes.h"
 #include "systable.h"
+#include "streamexecutorInt.h"
 #include "tname.h"
 
 #include "tdatablock.h"
@@ -2426,10 +2427,13 @@ void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
   if (!pInfo->pState) {
     return;
   }
-  void* pBuf = NULL;
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
+    void* pBuf = NULL;
-  pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
+    int32_t len = streamScanOperatorEncode(pInfo, &pBuf);
-  taosMemoryFree(pBuf);
+    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
+    taosMemoryFree(pBuf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 // other properties are recovered from the execution plan
@@ -2582,6 +2586,7 @@ FETCH_NEXT_BLOCK:
       case STREAM_NORMAL:
       case STREAM_GET_ALL:
         printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+        setStreamOperatorState(&pInfo->basic, pBlock->info.type);
         return pBlock;
       case STREAM_RETRIEVE: {
         pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
@@ -2622,6 +2627,7 @@ FETCH_NEXT_BLOCK:
 
           if (pInfo->pDeleteDataRes->info.rows > 0) {
             printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
+            setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
             return pInfo->pDeleteDataRes;
           } else {
             goto FETCH_NEXT_BLOCK;
@@ -2639,6 +2645,7 @@ FETCH_NEXT_BLOCK:
           if (pInfo->pDeleteDataRes->info.rows > 0) {
             pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
             printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result", GET_TASKID(pTaskInfo));
+            setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
             return pInfo->pDeleteDataRes;
           } else {
             goto FETCH_NEXT_BLOCK;
@@ -2652,6 +2659,7 @@ FETCH_NEXT_BLOCK:
         break;
     }
     printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
     return pBlock;
   } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
     qDebug("stream scan mode:%d, %s", pInfo->scanMode, id);
@@ -2659,6 +2667,7 @@ FETCH_NEXT_BLOCK:
       case STREAM_SCAN_FROM_RES: {
         pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
         doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey, pInfo->pRes);
+        setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
         doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
         pInfo->pRes->info.dataLoad = 1;
         blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
@@ -2762,6 +2771,7 @@ FETCH_NEXT_BLOCK:
         }
 
         doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes);
+        setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
         doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
         blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
 

source/libs/executor/src/streamcountwindowoperator.c

@@ -17,6 +17,7 @@
 #include "functionMgt.h"
 #include "operator.h"
 #include "querytask.h"
+#include "streamexecutorInt.h"
 #include "tchecksum.h"
 #include "tcommon.h"
 #include "tdatablock.h"
@@ -415,13 +416,16 @@ void* doStreamCountDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
 
 void doStreamCountSaveCheckpoint(SOperatorInfo* pOperator) {
   SStreamCountAggOperatorInfo* pInfo = pOperator->info;
-  int32_t                        len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  void*                          buf = taosMemoryCalloc(1, len);
+    int32_t len = doStreamCountEncodeOpState(NULL, 0, pOperator, true);
-  void*                          pBuf = buf;
+    void*   buf = taosMemoryCalloc(1, len);
-  len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
+    void*   pBuf = buf;
-  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
+    len = doStreamCountEncodeOpState(&pBuf, len, pOperator, true);
-                                                     strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
+    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_COUNT_OP_CHECKPOINT_NAME,
-  taosMemoryFree(buf);
+                                                       strlen(STREAM_COUNT_OP_CHECKPOINT_NAME), buf, len);
+    taosMemoryFree(buf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 void doResetCountWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock) {
@@ -550,6 +554,7 @@ static SSDataBlock* doStreamCountAgg(SOperatorInfo* pOperator) {
       break;
     }
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) {
       bool add = pInfo->destHasPrimaryKey && IS_NORMAL_COUNT_OP(pOperator);

source/libs/executor/src/streameventwindowoperator.c

@@ -18,6 +18,7 @@
 #include "functionMgt.h"
 #include "operator.h"
 #include "querytask.h"
+#include "streamexecutorInt.h"
 #include "tchecksum.h"
 #include "tcommon.h"
 #include "tcompare.h"
@@ -458,13 +459,16 @@ void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
 
 void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) {
   SStreamEventAggOperatorInfo* pInfo = pOperator->info;
-  int32_t                      len = doStreamEventEncodeOpState(NULL, 0, pOperator);
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  void*                        buf = taosMemoryCalloc(1, len);
+    int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator);
-  void*                        pBuf = buf;
+    void*   buf = taosMemoryCalloc(1, len);
-  len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
+    void*   pBuf = buf;
-  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
+    len = doStreamEventEncodeOpState(&pBuf, len, pOperator);
-                                                     strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);
+    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME,
-  taosMemoryFree(buf);
+                                                       strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len);
+    taosMemoryFree(buf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 static SSDataBlock* buildEventResult(SOperatorInfo* pOperator) {
@@ -531,6 +535,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) {
       break;
     }
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
         pBlock->info.type == STREAM_CLEAR) {

source/libs/executor/src/streamexecutorInt.c

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "executorInt.h"
+
+void setStreamOperatorState(SSteamOpBasicInfo* pBasicInfo, EStreamType type) {
+  if (type != STREAM_GET_ALL && type != STREAM_CHECKPOINT) {
+    pBasicInfo->updateOperatorInfo = true;
+  }
+}
+
+bool needSaveStreamOperatorInfo(SSteamOpBasicInfo* pBasicInfo) {
+  return pBasicInfo->updateOperatorInfo;
+}
+
+void saveStreamOperatorStateComplete(SSteamOpBasicInfo* pBasicInfo) {
+  pBasicInfo->updateOperatorInfo = false;
+}

source/libs/executor/src/streamtimewindowoperator.c

@@ -18,6 +18,7 @@
 #include "functionMgt.h"
 #include "operator.h"
 #include "querytask.h"
+#include "streamexecutorInt.h"
 #include "tchecksum.h"
 #include "tcommon.h"
 #include "tcompare.h"
@@ -1211,13 +1212,16 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
 
 void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {
   SStreamIntervalOperatorInfo* pInfo = pOperator->info;
-  int32_t                      len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  void*                        buf = taosMemoryCalloc(1, len);
+    int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);
-  void*                        pBuf = buf;
+    void*   buf = taosMemoryCalloc(1, len);
-  len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
+    void*   pBuf = buf;
-  pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
+    len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);
-                                        strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
+    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,
-  taosMemoryFree(buf);
+                                          strlen(STREAM_INTERVAL_OP_CHECKPOINT_NAME), buf, len);
+    taosMemoryFree(buf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 static void copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {
@@ -1347,6 +1351,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
     }
     pInfo->numOfDatapack++;
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
       pInfo->binfo.pRes->info.type = pBlock->info.type;
@@ -2690,13 +2695,16 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera
 
 void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {
   SStreamSessionAggOperatorInfo* pInfo = pOperator->info;
-  int32_t                        len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  void*                          buf = taosMemoryCalloc(1, len);
+    int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);
-  void*                          pBuf = buf;
+    void*   buf = taosMemoryCalloc(1, len);
-  len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
+    void*   pBuf = buf;
-  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
+    len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);
-                                                     strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
+    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,
-  taosMemoryFree(buf);
+                                                       strlen(STREAM_SESSION_OP_CHECKPOINT_NAME), buf, len);
+    taosMemoryFree(buf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 void resetUnCloseSessionWinInfo(SSHashObj* winMap) {
@@ -2766,6 +2774,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
       break;
     }
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
         pBlock->info.type == STREAM_CLEAR) {
@@ -3176,6 +3185,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {
       break;
     }
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
         pBlock->info.type == STREAM_CLEAR) {
@@ -3673,13 +3683,16 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato
 
 void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {
   SStreamStateAggOperatorInfo* pInfo = pOperator->info;
-  int32_t                      len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
+  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
-  void*                        buf = taosMemoryCalloc(1, len);
+    int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);
-  void*                        pBuf = buf;
+    void*   buf = taosMemoryCalloc(1, len);
-  len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
+    void*   pBuf = buf;
-  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
+    len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);
-                                                     strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);
+    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,
-  taosMemoryFree(buf);
+                                                       strlen(STREAM_STATE_OP_CHECKPOINT_NAME), buf, len);
+    taosMemoryFree(buf);
+    saveStreamOperatorStateComplete(&pInfo->basic);
+  }
 }
 
 static SSDataBlock* buildStateResult(SOperatorInfo* pOperator) {
@@ -3746,6 +3759,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
       break;
     }
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
         pBlock->info.type == STREAM_CLEAR) {
@@ -4069,6 +4083,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
     pInfo->numOfDatapack++;
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||
         pBlock->info.type == STREAM_CLEAR) {
@@ -4465,6 +4480,7 @@ static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {
     }
     pInfo->numOfDatapack++;
     printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));
+    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
 
     if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {
       pInfo->binfo.pRes->info.type = pBlock->info.type;

source/libs/executor/src/sysscanoperator.c

@@ -549,7 +549,7 @@ static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) {
 
   if (!pInfo->pCur || !pInfo->pSchema) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
-    qError("sysTableScanUserCols failed since %s", terrstr(terrno));
+    qError("sysTableScanUserCols failed since %s", terrstr());
     blockDataDestroy(pDataBlock);
     pInfo->loadInfo.totalRows = 0;
     return NULL;
@@ -1651,50 +1651,56 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
   SSysTableScanInfo* pInfo = pOperator->info;
   char               dbName[TSDB_DB_NAME_LEN] = {0};
 
-  if (isTaskKilled(pOperator->pTaskInfo)) {
+  while (1) {
-    setOperatorCompleted(pOperator);
-    return NULL;
-  }
 
-  blockDataCleanup(pInfo->pRes);
+    if (isTaskKilled(pOperator->pTaskInfo)) {
-
-  const char* name = tNameGetTableName(&pInfo->name);
-  if (pInfo->showRewrite) {
-    getDBNameFromCondition(pInfo->pCondition, dbName);
-    if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
-        strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
-      sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
-    }
-  } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
-    getDBNameFromCondition(pInfo->pCondition, dbName);
-    if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
-    sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
-  }
-
-  SSDataBlock* pBlock = NULL;
-  if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
-    pBlock = sysTableScanUserTables(pOperator);
-  } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
-    pBlock = sysTableScanUserTags(pOperator);
-  } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) {
-    pBlock = sysTableScanUserCols(pOperator);
-  } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
-             IS_SYS_DBNAME(dbName)) {
-    pBlock = sysTableScanUserSTables(pOperator);
-  } else {  // load the meta from mnode of the given epset
-    pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
-  }
-
-  sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
-  if (pBlock != NULL) {
-    bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
-    if (limitReached) {
       setOperatorCompleted(pOperator);
+      return NULL;
     }
 
-    return pBlock->info.rows > 0 ? pBlock : NULL;
+    blockDataCleanup(pInfo->pRes);
-  } else {
+
-    return NULL;
+    const char* name = tNameGetTableName(&pInfo->name);
+    if (pInfo->showRewrite) {
+      getDBNameFromCondition(pInfo->pCondition, dbName);
+      if (strncasecmp(name, TSDB_INS_TABLE_COMPACTS, TSDB_TABLE_FNAME_LEN) != 0 &&
+          strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, TSDB_TABLE_FNAME_LEN) != 0) {
+        sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
+      }
+    } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) {
+      getDBNameFromCondition(pInfo->pCondition, dbName);
+      if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
+      sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb);
+    }
+
+    SSDataBlock* pBlock = NULL;
+    if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
+      pBlock = sysTableScanUserTables(pOperator);
+    } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
+      pBlock = sysTableScanUserTags(pOperator);
+    } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) {
+      pBlock = sysTableScanUserCols(pOperator);
+    } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
+              IS_SYS_DBNAME(dbName)) {
+      pBlock = sysTableScanUserSTables(pOperator);
+    } else {  // load the meta from mnode of the given epset
+      pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
+    }
+
+    sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
+    if (pBlock != NULL) {
+      bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
+      if (limitReached) {
+        setOperatorCompleted(pOperator);
+      }
+
+      if (pBlock->info.rows == 0) {
+        continue;
+      }
+      return pBlock;
+    } else {
+      return NULL;
+    }
   }
 }
 

source/libs/executor/src/timesliceoperator.c

@@ -204,6 +204,11 @@ static bool checkDuplicateTimestamps(STimeSliceOperatorInfo* pSliceInfo, SColumn
   SRowKey cur = {.ts = currentTs, .numOfPKs = (pPkCol != NULL)? 1:0};
   if (pPkCol != NULL) {
     cur.pks[0].type = pPkCol->info.type;
+    if (IS_VAR_DATA_TYPE(pPkCol->info.type)) {
+      cur.pks[0].pData = (uint8_t*)colDataGetVarData(pPkCol, curIndex);
+    } else {
+      memcpy(&cur.pks[0].val, colDataGetData(pPkCol, curIndex), pPkCol->info.bytes);
+    }
   }
 
   // let's discard the duplicated ts

source/libs/function/src/udfd.c

@@ -842,7 +842,7 @@ void udfdGetFuncBodyPath(const SUdf *udf, char *path) {
 int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
   if (!osDataSpaceAvailable()) {
     terrno = TSDB_CODE_NO_DISKSPACE;
-    fnError("udfd create shared library failed since %s", terrstr(terrno));
+    fnError("udfd create shared library failed since %s", terrstr());
     return terrno;
   }
 
@@ -1424,7 +1424,7 @@ int main(int argc, char *argv[]) {
     printf("failed to start since init log error\n");
   }
 
-  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
     fnError("failed to start since read config error");
     return -2;
   }

source/libs/function/test/runUdf.c

@@ -127,7 +127,7 @@ int aggregateFuncTest() {
 int main(int argc, char *argv[]) {
   parseArgs(argc, argv);
   initLog();
-  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0, true) != 0) {
     fnError("failed to start since read config error");
     return -1;
   }

source/libs/geometry/src/geosWrapper.c

@@ -86,6 +86,67 @@ _exit:
   return code;
 }
 
+static int initWktRegex(pcre2_code **ppRegex, pcre2_match_data **ppMatchData) {
+  int   ret = 0;
+  char *wktPatternWithSpace = taosMemoryCalloc(4, 1024);
+  sprintf(
+      wktPatternWithSpace,
+      "^( *)point( *)z?m?( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\)))|linestring( *)z?m?( "
+      "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))|polygon( *)z?m?( "
+      "*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
+      "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
+      "*)\\)))|multipoint( *)z?m?( *)((empty)|(\\(( "
+      "*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))(( *)(,)( "
+      "*)((([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}|((empty)|(\\(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *)\\))))( *))*( "
+      "*)\\)))|multilinestring( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
+      "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( "
+      "*)\\)))|multipolygon( *)z?m?( *)((empty)|(\\(( *)((empty)|(\\(( *)((empty)|(\\(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
+      "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))(( *)(,)( "
+      "*)((empty)|(\\(( *)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))(( *)(,)( "
+      "*)((empty)|(\\(( *)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}(( *)(,)( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?(( "
+      "*)(([-+]?[0-9]+\\.?[0-9]*)|([-+]?[0-9]*\\.?[0-9]+))(e[-+]?[0-9]+)?){1,3}( *))*( *)\\)))( *))*( *)\\)))( *))*( "
+      "*)\\)))|(GEOCOLLECTION\\((?R)(( *)(,)( *)(?R))*( *)\\))( *)$");
+
+  ret = doRegComp(ppRegex, ppMatchData, wktPatternWithSpace);
+  taosMemoryFree(wktPatternWithSpace);
+  return ret;
+}
+
 int32_t initCtxGeomFromText() {
   int32_t code = TSDB_CODE_FAILED;
   SGeosContext* geosCtx = getThreadLocalGeosCtx();
@@ -113,6 +174,10 @@ int32_t initCtxGeomFromText() {
     }
   }
 
+  if (geosCtx->WKTRegex == NULL) {
+    if (initWktRegex(&geosCtx->WKTRegex, &geosCtx->WKTMatchData) != 0) return code;
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -125,6 +190,11 @@ int32_t doGeomFromText(const char *inputWKT, unsigned char **outputGeom, size_t
   GEOSGeometry *geom = NULL;
   unsigned char *wkb = NULL;
 
+  if (doRegExec(inputWKT, geosCtx->WKTRegex, geosCtx->WKTMatchData) != 0) {
+     code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;
+     goto _exit;
+  }
+
   geom = GEOSWKTReader_read_r(geosCtx->handle, geosCtx->WKTReader, inputWKT);
   if (geom == NULL) {
     code = TSDB_CODE_FUNC_FUNTION_PARA_VALUE;

source/libs/monitorfw/CMakeLists.txt

@@ -5,5 +5,7 @@ target_include_directories(
   PUBLIC "${TD_SOURCE_DIR}/include/libs/monitorfw"
   PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
-
+if(${TD_DARWIN})
+    target_compile_options(monitorfw PRIVATE -Wno-error=deprecated-pragma)
+endif(${TD_DARWIN})
 target_link_libraries(monitorfw os util common transport)

source/libs/nodes/src/nodesCodeFuncs.c

@@ -187,6 +187,8 @@ const char* nodesNodeName(ENodeType type) {
       return "BalanceVgroupStmt";
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
       return "BalanceVgroupLeaderStmt";
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
+      return "BalanceVgroupLeaderStmt";
     case QUERY_NODE_MERGE_VGROUP_STMT:
       return "MergeVgroupStmt";
     case QUERY_NODE_SHOW_DB_ALIVE_STMT:
@@ -7607,6 +7609,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
       return TSDB_CODE_SUCCESS;  // SBalanceVgroupStmt has no fields to serialize.
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
       return TSDB_CODE_SUCCESS;  // SBalanceVgroupLeaderStmt has no fields to serialize.
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
+      return TSDB_CODE_SUCCESS;
     case QUERY_NODE_MERGE_VGROUP_STMT:
       return mergeVgroupStmtToJson(pObj, pJson);
     case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@@ -7953,7 +7957,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
     case QUERY_NODE_BALANCE_VGROUP_STMT:
       return TSDB_CODE_SUCCESS;  // SBalanceVgroupStmt has no fields to deserialize.
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
-      return TSDB_CODE_SUCCESS;  // SBalanceVgroupLeaderStmt has no fields to deserialize.
+      return TSDB_CODE_SUCCESS;
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
+      return TSDB_CODE_SUCCESS;   // SBalanceVgroupLeaderStmt has no fields to deserialize.
     case QUERY_NODE_MERGE_VGROUP_STMT:
       return jsonToMergeVgroupStmt(pJson, pObj);
     case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:

source/libs/nodes/src/nodesUtilFuncs.c

@@ -473,6 +473,8 @@ SNode* nodesMakeNode(ENodeType type) {
       return makeNode(type, sizeof(SBalanceVgroupStmt));
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
       return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
+      return makeNode(type, sizeof(SBalanceVgroupLeaderStmt));
     case QUERY_NODE_MERGE_VGROUP_STMT:
       return makeNode(type, sizeof(SMergeVgroupStmt));
     case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:
@@ -1161,6 +1163,7 @@ void nodesDestroyNode(SNode* pNode) {
     case QUERY_NODE_RESUME_STREAM_STMT:          // no pointer field
     case QUERY_NODE_BALANCE_VGROUP_STMT:         // no pointer field
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:  // no pointer field
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:  // no pointer field
     case QUERY_NODE_MERGE_VGROUP_STMT:           // no pointer field
       break;
     case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT:

source/libs/parser/inc/parAst.h

@@ -274,6 +274,7 @@ SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId
 SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
 SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
 SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId);
+SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName);
 SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
 SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
 SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);

source/libs/parser/inc/sql.y

@@ -773,6 +773,7 @@ cmd ::= KILL COMPACT NK_INTEGER(A).
 /************************************************ merge/redistribute/ vgroup ******************************************/
 cmd ::= BALANCE VGROUP.                                                           { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
 cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A).                                    { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); }
+cmd ::= BALANCE VGROUP LEADER DATABASE db_name(A).                                { pCxt->pRootNode = createBalanceVgroupLeaderDBNameStmt(pCxt, &A); }
 cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B).                                 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
 cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B).                          { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
 cmd ::= SPLIT VGROUP NK_INTEGER(A).                                               { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }

source/libs/parser/src/parAstCreater.c

@@ -2809,6 +2809,16 @@ SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgI
   return (SNode*)pStmt;
 }
 
+SNode* createBalanceVgroupLeaderDBNameStmt(SAstCreateContext* pCxt, const SToken* pDbName){
+  CHECK_PARSER_STATUS(pCxt);
+  SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT);
+  CHECK_OUT_OF_MEM(pStmt);
+  if (NULL != pDbName) {
+    COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName);
+  }
+  return (SNode*)pStmt;
+}
+
 SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2) {
   CHECK_PARSER_STATUS(pCxt);
   SMergeVgroupStmt* pStmt = (SMergeVgroupStmt*)nodesMakeNode(QUERY_NODE_MERGE_VGROUP_STMT);

source/libs/parser/src/parAstParser.c

@@ -42,7 +42,6 @@ int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placehol
   return TSDB_CODE_SUCCESS;
 }
 
-
 int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
   SAstCreateContext cxt;
   initAstCreateContext(pParseCxt, &cxt);
@@ -534,6 +533,10 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt*
 static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
   int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TAGS,
                                          pCxt->pMetaCache);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal,
+                                   ((SValueNode*)pStmt->pTbName)->literal, pCxt->pMetaCache);
+  }
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache);
   }
@@ -631,8 +634,8 @@ static int32_t collectMetaKeyFromShowCompacts(SCollectMetaKeyCxt* pCxt, SShowStm
 }
 
 static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
-  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_COMPACT_DETAILS,
+  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB,
-                                         pCxt->pMetaCache);
+                                         TSDB_INS_TABLE_COMPACT_DETAILS, pCxt->pMetaCache);
   return code;
 }
 
@@ -687,7 +690,7 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
   int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0);
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
-                                  AUTH_TYPE_READ, pCxt->pMetaCache);
+                                      AUTH_TYPE_READ, pCxt->pMetaCache);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
@@ -696,7 +699,6 @@ static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowC
   return code;
 }
 
-
 static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
   return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS,
                                  pCxt->pMetaCache);
@@ -755,24 +757,22 @@ static int32_t collectMetaKeyFromRevoke(SCollectMetaKeyCxt* pCxt, SRevokeStmt* p
   return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
 }
 
-
 static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreateViewStmt* pStmt) {
-  int32_t code =
+  int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
-      reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
   if (TSDB_CODE_SUCCESS == code) {
     code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE,
                                   pCxt->pMetaCache);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
+    code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
-                                  pCxt->pMetaCache);
+                                      AUTH_TYPE_ALTER, pCxt->pMetaCache);
   }
   return code;
 }
 
 static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) {
-  int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
+  int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName,
-                                  pCxt->pMetaCache);
+                                            pStmt->viewName, AUTH_TYPE_ALTER, pCxt->pMetaCache);
   return code;
 }
 
@@ -783,7 +783,7 @@ static int32_t collectMetaKeyFromCreateTSMAStmt(SCollectMetaKeyCxt* pCxt, SCreat
     code = reserveTSMAInfoInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
     if (TSDB_CODE_SUCCESS == code) {
       char dstTbName[TSDB_TABLE_NAME_LEN] = {0};
-      snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s"TSMA_RES_STB_POSTFIX, pStmt->tableName);
+      snprintf(dstTbName, TSDB_TABLE_NAME_LEN, "%s" TSMA_RES_STB_POSTFIX, pStmt->tableName);
       code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);
       if (TSDB_CODE_SUCCESS == code) {
         code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, dstTbName, pCxt->pMetaCache);
@@ -921,7 +921,7 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
     case QUERY_NODE_SHOW_COMPACTS_STMT:
       return collectMetaKeyFromShowCompacts(pCxt, (SShowStmt*)pStmt);
     case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
-      return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);               
+      return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);
     case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
       return collectMetaKeyFromShowGrantsFull(pCxt, (SShowStmt*)pStmt);
     case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:

source/libs/parser/src/parInsertUtil.c

@@ -677,6 +677,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
   p += sizeof(uint64_t);
 
   int8_t* fields = p;
+  if(*fields >= TSDB_DATA_TYPE_MAX || *fields < 0){
+    uError("fields type error:%d", *fields);
+    ret = TSDB_CODE_INVALID_PARA;
+    goto end;
+  }
   p += numOfCols * (sizeof(int8_t) + sizeof(int32_t));
 
   int32_t* colLength = (int32_t*)p;
@@ -688,12 +693,12 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
   SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo;
 
   if (tFields != NULL && numFields != numOfCols) {
-    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d != raw numOfCols:%d", numFields, numOfCols);
+    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d not equal to data cols:%d", numFields, numOfCols);
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
   if (tFields != NULL && numFields > boundInfo->numOfBound) {
-    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d > boundInfo->numOfBound:%d", numFields, boundInfo->numOfBound);
+    if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound);
     ret = TSDB_CODE_INVALID_PARA;
     goto end;
   }
@@ -702,8 +707,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
       SSchema*  pColSchema = &pSchema[j];
       SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j);
       if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
-        if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
+        if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
-                                     pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
+                                     pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
         ret = TSDB_CODE_INVALID_PARA;
         goto end;
       }
@@ -732,8 +737,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
         SSchema* pColSchema = &pSchema[j];
         if (strcmp(pColSchema->name, tFields[i].name) == 0) {
           if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) {
-            if (errstr != NULL) snprintf(errstr, errstrLen, "type or bytes not equal, id:%d, type:%d, raw type:%d. bytes:%d, raw bytes:%d",
+            if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d",
-                                         pColSchema->colId, pColSchema->type, *fields, pColSchema->bytes, *(int32_t*)(fields + sizeof(int8_t)));
+                                         pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t)));
             ret = TSDB_CODE_INVALID_PARA;
             goto end;
           }

source/libs/parser/src/parTranslater.c

@@ -4338,7 +4338,7 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinPare
           return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
         }
 #ifdef TD_ENTERPRISE
-        if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) {
+        if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType && !pCurrSmt->tagScan) {
           return translateView(pCxt, pTable, &name);
         }
         translateAudit(pCxt, pRealTable, &name);
@@ -10578,6 +10578,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
 static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
   SBalanceVgroupLeaderReq req = {0};
   req.vgId = pStmt->vgId;
+  if(pStmt->dbName != NULL) strcpy(req.db, pStmt->dbName);
   int32_t code =
       buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
   tFreeSBalanceVgroupLeaderReq(&req);
@@ -11263,6 +11264,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
     case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT:
       code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
       break;
+    case QUERY_NODE_BALANCE_VGROUP_LEADER_DATABASE_STMT:
+      code = translateBalanceVgroupLeader(pCxt, (SBalanceVgroupLeaderStmt*)pNode);
+      break;
     case QUERY_NODE_MERGE_VGROUP_STMT:
       code = translateMergeVgroup(pCxt, (SMergeVgroupStmt*)pNode);
       break;
@@ -11867,6 +11871,37 @@ static int32_t rewriteShowVgroups(STranslateContext* pCxt, SQuery* pQuery) {
   return code;
 }
 
+static int32_t checkShowTags(STranslateContext* pCxt, const SShowStmt* pShow) {
+  int32_t     code = 0;
+  SName       name;
+  STableMeta* pTableMeta = NULL;
+  code = getTargetMeta(pCxt,
+                       toName(pCxt->pParseCxt->acctId, ((SValueNode*)pShow->pDbName)->literal,
+                              ((SValueNode*)pShow->pTbName)->literal, &name),
+                       &pTableMeta, true);
+  if (TSDB_CODE_SUCCESS != code) {
+    code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
+    goto _exit;
+  }
+  if (TSDB_SUPER_TABLE != pTableMeta->tableType && TSDB_CHILD_TABLE != pTableMeta->tableType) {
+    code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAGS_PC,
+                                "The _TAGS pseudo column can only be used for child table and super table queries");
+    goto _exit;
+  }
+
+_exit:
+  taosMemoryFreeClear(pTableMeta);
+  return code;
+}
+
+static int32_t rewriteShowTags(STranslateContext* pCxt, SQuery* pQuery) {
+  int32_t code = checkShowTags(pCxt, (SShowStmt*)pQuery->pRoot);
+  if (TSDB_CODE_SUCCESS == code) {
+    code = rewriteShow(pCxt, pQuery);
+  }
+  return code;
+}
+
 static SNode* createTagsFunction() {
   SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
   if (NULL == pFunc) {
@@ -13170,7 +13205,6 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
     case QUERY_NODE_SHOW_APPS_STMT:
     case QUERY_NODE_SHOW_CONSUMERS_STMT:
     case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
-    case QUERY_NODE_SHOW_TAGS_STMT:
     case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
     case QUERY_NODE_SHOW_VIEWS_STMT:
     case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
@@ -13181,6 +13215,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
     case QUERY_NODE_SHOW_TSMAS_STMT:
       code = rewriteShow(pCxt, pQuery);
       break;
+    case QUERY_NODE_SHOW_TAGS_STMT:
+      code = rewriteShowTags(pCxt, pQuery);
+      break;
     case QUERY_NODE_SHOW_VGROUPS_STMT:
       code = rewriteShowVgroups(pCxt, pQuery);
       break;

source/libs/planner/src/planOptimizer.c

@@ -246,11 +246,10 @@ static bool scanPathOptMayBeOptimized(SLogicNode* pNode) {
 
 static bool scanPathOptShouldGetFuncs(SLogicNode* pNode) {
   if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
-    if (pNode->pParent && QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) {
+    if (!pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) ||
-      if (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType) return true;
+        WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType)
-    } else {
       return !scanPathOptHaveNormalCol(((SPartitionLogicNode*)pNode)->pPartitionKeys);
-    }
+    return false;
   }
 
   if ((QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) &&

source/libs/stream/src/streamBackendRocksdb.c

@@ -1588,7 +1588,7 @@ int32_t valueEncode(void* value, int32_t vlen, int64_t ttl, char** dest) {
   if (*dest == NULL) {
     size_t size = sizeof(key.unixTimestamp) + sizeof(key.len) + sizeof(key.rawLen) + sizeof(key.compress) + key.len;
     char*  p = taosMemoryCalloc(1, size);
-    char* buf = p;
+    char*  buf = p;
     len += taosEncodeFixedI64((void**)&buf, key.unixTimestamp);
     len += taosEncodeFixedI32((void**)&buf, key.len);
     len += taosEncodeFixedI32((void**)&buf, key.rawLen);
@@ -2740,8 +2740,10 @@ int32_t streamStateGetGroupKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey,
     if (pKey->groupId == groupId) {
       return 0;
     }
-    taosMemoryFree((void*)*pVal);
+    if (pVal != NULL) {
-    *pVal = NULL;
+      taosMemoryFree((void*)*pVal);
+      *pVal = NULL;
+    }
   }
   return -1;
 }

source/libs/stream/src/streamMeta.c

@@ -1631,6 +1631,8 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t ta
       streamMetaAddTaskLaunchResult(pMeta, hId.streamId, hId.taskId, startTs, now, false);
     }
   } else {
+    streamMetaRUnLock(pMeta);
+
     stError("failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
             streamId, taskId, pMeta->vgId);
     code = TSDB_CODE_STREAM_TASK_NOT_EXIST;

source/libs/stream/src/tstreamFileState.c

@@ -557,7 +557,6 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
   const int32_t BATCH_LIMIT = 256;
 
   int64_t    st = taosGetTimestampMs();
-  int32_t    numOfElems = listNEles(pSnapshot);
   SListNode* pNode = NULL;
 
   int idx = streamStateGetCfIdx(pFileState->pFileStore, pFileState->cfName);
@@ -589,8 +588,11 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
   }
   taosMemoryFree(buf);
 
-  if (streamStateGetBatchSize(batch) > 0) {
+  int32_t numOfElems = streamStateGetBatchSize(batch);
+  if (numOfElems > 0) {
     streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
+  } else {
+    goto _end;
   }
 
   streamStateClearBatch(batch);
@@ -609,6 +611,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot,
     streamStatePutBatch_rocksdb(pFileState->pFileStore, batch);
   }
 
+_end:
   streamStateDestroyBatch(batch);
   return code;
 }

source/libs/sync/src/syncMain.c

@@ -463,7 +463,7 @@ _DEL_WAL:
                   pSyncNode->snapshottingIndex, lastApplyIndex);
         } else {
           sNError(pSyncNode, "wal snapshot begin error since:%s, index:%" PRId64 ", last apply index:%" PRId64,
-                  terrstr(terrno), pSyncNode->snapshottingIndex, lastApplyIndex);
+                  terrstr(), pSyncNode->snapshottingIndex, lastApplyIndex);
           atomic_store_64(&pSyncNode->snapshottingIndex, SYNC_INDEX_INVALID);
         }
 

source/libs/transport/inc/transComm.h

@@ -123,7 +123,7 @@ typedef struct {
   STransMsg* pRsp;
   SEpSet     epSet;
   int8_t     hasEpSet;
-  tsem_t*    pSem;
+  tsem2_t*   pSem;
   int8_t     inited;
   SRWLatch   latch;
 } STransSyncMsg;
@@ -302,8 +302,8 @@ int  transClearBuffer(SConnBuffer* buf);
 int  transDestroyBuffer(SConnBuffer* buf);
 int  transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
 bool transReadComplete(SConnBuffer* connBuf);
-int  transResetBuffer(SConnBuffer* connBuf);
+int  transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf);
-int  transDumpFromBuffer(SConnBuffer* connBuf, char** buf);
+int  transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf);
 
 int transSetConnOption(uv_tcp_t* stream, int keepalive);
 

source/libs/transport/src/transCli.c

@@ -382,13 +382,18 @@ void cliHandleResp(SCliConn* conn) {
 
   STransMsgHead* pHead = NULL;
 
-  int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead);
+  int8_t  resetBuf = conn->status == ConnAcquire ? 0 : 1;
+  int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead, resetBuf);
   if (msgLen <= 0) {
     taosMemoryFree(pHead);
     tDebug("%s conn %p recv invalid packet ", CONN_GET_INST_LABEL(conn), conn);
     return;
   }
 
+  if (resetBuf == 0) {
+    tTrace("%s conn %p not reset read buf", transLabel(pTransInst), conn);
+  }
+
   if (transDecompressMsg((char**)&pHead, msgLen) < 0) {
     tDebug("%s conn %p recv invalid packet, failed to decompress", CONN_GET_INST_LABEL(conn), conn);
   }
@@ -2450,7 +2455,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
           pSyncMsg->hasEpSet = 1;
           epsetAssign(&pSyncMsg->epSet, &pCtx->epSet);
         }
-        tsem_post(pSyncMsg->pSem);
+        tsem2_post(pSyncMsg->pSem);
         taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef);
       } else {
         rpcFreeCont(pResp->pCont);
@@ -2679,8 +2684,8 @@ _RETURN:
   return ret;
 }
 int64_t transCreateSyncMsg(STransMsg* pTransMsg) {
-  tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t));
+  tsem2_t* sem = taosMemoryCalloc(1, sizeof(tsem2_t));
-  tsem_init(sem, 0, 0);
+  tsem2_init(sem, 0, 0);
 
   STransSyncMsg* pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg));
 
@@ -2740,7 +2745,7 @@ int transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STr
     goto _RETURN;
   }
 
-  ret = tsem_timewait(pSyncMsg->pSem, timeoutMs);
+  ret = tsem2_timewait(pSyncMsg->pSem, timeoutMs);
   if (ret < 0) {
     pRsp->code = TSDB_CODE_TIMEOUT_ERROR;
     ret = TSDB_CODE_TIMEOUT_ERROR;

source/libs/transport/src/transComm.c

@@ -126,7 +126,7 @@ int transClearBuffer(SConnBuffer* buf) {
   return 0;
 }
 
-int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
+int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) {
   static const int HEADSIZE = sizeof(STransMsgHead);
 
   SConnBuffer* p = connBuf;
@@ -137,7 +137,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
   if (total >= HEADSIZE && !p->invalid) {
     *buf = taosMemoryCalloc(1, total);
     memcpy(*buf, p->buf, total);
-    if (transResetBuffer(connBuf) < 0) {
+    if (transResetBuffer(connBuf, resetBuf) < 0) {
       return -1;
     }
   } else {
@@ -146,7 +146,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
   return total;
 }
 
-int transResetBuffer(SConnBuffer* connBuf) {
+int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) {
   SConnBuffer* p = connBuf;
   if (p->total < p->len) {
     int left = p->len - p->total;
@@ -159,8 +159,10 @@ int transResetBuffer(SConnBuffer* connBuf) {
     p->total = 0;
     p->len = 0;
     if (p->cap > BUFFER_CAP) {
-      p->cap = BUFFER_CAP;
+      if (resetBuf) {
-      p->buf = taosMemoryRealloc(p->buf, p->cap);
+        p->cap = BUFFER_CAP;
+        p->buf = taosMemoryRealloc(p->buf, p->cap);
+      }
     }
   } else {
     ASSERTS(0, "invalid read from sock buf");
@@ -681,7 +683,7 @@ void transDestroySyncMsg(void* msg) {
   if (msg == NULL) return;
 
   STransSyncMsg* pSyncMsg = msg;
-  tsem_destroy(pSyncMsg->pSem);
+  tsem2_destroy(pSyncMsg->pSem);
   taosMemoryFree(pSyncMsg->pSem);
   transFreeMsg(pSyncMsg->pRsp->pCont);
   taosMemoryFree(pSyncMsg->pRsp);

source/libs/transport/src/transSvr.c

@@ -342,11 +342,15 @@ static bool uvHandleReq(SSvrConn* pConn) {
 
   STransMsgHead* pHead = NULL;
 
-  int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead);
+  int8_t resetBuf = pConn->status == ConnAcquire ? 0 : 1;
+  int    msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead, resetBuf);
   if (msgLen <= 0) {
     tError("%s conn %p read invalid packet", transLabel(pTransInst), pConn);
     return false;
   }
+  if (resetBuf == 0) {
+    tTrace("%s conn %p not reset read buf", transLabel(pTransInst), pConn);
+  }
 
   if (transDecompressMsg((char**)&pHead, msgLen) < 0) {
     tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn);
@@ -676,7 +680,8 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
   taosMemoryFree(smsg);
 }
 static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); }
-static void              destroyAllConn(SWorkThrd* pThrd) {
+
+static void destroyAllConn(SWorkThrd* pThrd) {
   tTrace("thread %p destroy all conn ", pThrd);
   while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
     queue* h = QUEUE_HEAD(&pThrd->conn);

source/os/src/osSemaphore.c

@@ -215,14 +215,6 @@ int32_t taosGetAppName(char* name, int32_t* len) {
   return 0;
 }
 
-int32_t tsem_wait(tsem_t* sem) {
-  int ret = 0;
-  do {
-    ret = sem_wait(sem);
-  } while (ret != 0 && errno == EINTR);
-  return ret;
-}
-
 int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
   int ret = 0;
 
@@ -241,4 +233,101 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) {
   return ret;
 }
 
+int32_t tsem_wait(tsem_t* sem) {
+  int ret = 0;
+  do {
+    ret = sem_wait(sem);
+  } while (ret != 0 && errno == EINTR);
+  return ret;
+}
+
+int tsem2_init(tsem2_t* sem, int pshared, unsigned int value) {
+  int ret = taosThreadMutexInit(&sem->mutex, NULL);
+  if (ret != 0) return ret;
+  ret = taosThreadCondAttrInit(&sem->attr);
+  if (ret != 0)
+  {
+    taosThreadMutexDestroy(&sem->mutex);
+    return ret;
+  }
+  ret = taosThreadCondAttrSetclock(&sem->attr, CLOCK_MONOTONIC);
+  if (ret != 0)
+  {
+    taosThreadMutexDestroy(&sem->mutex);
+    taosThreadCondAttrDestroy(&sem->attr);
+    return ret;
+  }
+  ret = taosThreadCondInit(&sem->cond, &sem->attr);
+  if (ret != 0)
+  {
+    taosThreadMutexDestroy(&sem->mutex);
+    taosThreadCondAttrDestroy(&sem->attr);
+    return ret;
+  }
+
+  sem->count = value;
+  return 0;
+}
+
+int tsem2_post(tsem2_t *sem) {
+  taosThreadMutexLock(&sem->mutex);
+  sem->count++;
+  taosThreadCondSignal(&sem->cond);
+  taosThreadMutexUnlock(&sem->mutex);
+  return 0;
+}
+
+int tsem2_destroy(tsem2_t* sem) {
+  taosThreadMutexDestroy(&sem->mutex);
+  taosThreadCondDestroy(&sem->cond);
+  taosThreadCondAttrDestroy(&sem->attr);
+  return 0;
+}
+
+int32_t tsem2_wait(tsem2_t* sem) {
+  taosThreadMutexLock(&sem->mutex);
+  while (sem->count <= 0) {
+    int ret = taosThreadCondWait(&sem->cond, &sem->mutex);
+    if (0 == ret) {
+      continue;
+    } else {
+      taosThreadMutexUnlock(&sem->mutex);
+      return ret;
+    }
+  }
+  sem->count--;
+  taosThreadMutexUnlock(&sem->mutex);
+  return 0;
+}
+
+int32_t tsem2_timewait(tsem2_t* sem, int64_t ms) {
+  int ret = 0;
+
+  taosThreadMutexLock(&sem->mutex);
+  if (sem->count <= 0) {
+    struct timespec ts = {0};
+    if (clock_gettime(CLOCK_MONOTONIC, &ts) == -1) {
+      taosThreadMutexUnlock(&sem->mutex);
+      return -1;
+    }
+
+    ts.tv_sec += ms / 1000;
+    ts.tv_nsec += (ms % 1000) * 1000000;
+    ts.tv_sec += ts.tv_nsec / 1000000000;
+    ts.tv_nsec %= 1000000000;
+
+    while (sem->count <= 0) {
+      ret = taosThreadCondTimedWait(&sem->cond, &sem->mutex, &ts);
+      if (ret != 0) {
+        taosThreadMutexUnlock(&sem->mutex);
+        return ret;
+      }
+    }
+  }
+
+  sem->count--;
+  taosThreadMutexUnlock(&sem->mutex);
+  return ret;
+}
+
 #endif

source/os/src/osThread.c

@@ -170,6 +170,16 @@ int32_t taosThreadCondAttrInit(TdThreadCondAttr *attr) {
 #endif
 }
 
+int32_t taosThreadCondAttrSetclock(TdThreadCondAttr *attr, int clockId) {
+#ifdef __USE_WIN_THREAD
+  return 0;
+#elif defined(__APPLE__)
+  return 0;
+#else
+  return pthread_condattr_setclock(attr, clockId);
+#endif
+}
+
 int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr *attr, int32_t pshared) {
 #ifdef __USE_WIN_THREAD
   return 0;